1cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen//===-- RegAllocGreedy.cpp - greedy register allocator --------------------===// 2cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// 3cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// The LLVM Compiler Infrastructure 4cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// 5cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// This file is distributed under the University of Illinois Open Source 6cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// License. See LICENSE.TXT for details. 7cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// 8cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen//===----------------------------------------------------------------------===// 9cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// 10cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// This file defines the RAGreedy function pass for register allocation in 11cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// optimized builds. 12cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen// 13cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen//===----------------------------------------------------------------------===// 14cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 15dd479e9769eceee9fcc34e2173376024f3aa3c5fJakob Stoklund Olesen#include "AllocationOrder.h" 165907d863659eb972ebb2afe07bc863a4c616f0efJakob Stoklund Olesen#include "InterferenceCache.h" 17cfafc54040cc9722995558124f253d05a038176bJakob Stoklund Olesen#include "LiveDebugVariables.h" 18cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "RegAllocBase.h" 19b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen#include "SpillPlacement.h" 20d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "Spiller.h" 21d0bb5e2ca05d9c942223bf05e5940bb0c6cc9d3fJakob Stoklund Olesen#include "SplitKit.h" 220db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund Olesen#include "llvm/ADT/Statistic.h" 23cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/Analysis/AliasAnalysis.h" 24cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/CalcSpillWeights.h" 25b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen#include "llvm/CodeGen/EdgeBundles.h" 26cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/LiveIntervalAnalysis.h" 27789d5d85ba6e9259a8e0f0bcfbd06a59ad164512Pete Cooper#include "llvm/CodeGen/LiveRangeEdit.h" 281ead68d769f27f6d68d4aaeffe4199fa2cacbc95Jakob Stoklund Olesen#include "llvm/CodeGen/LiveRegMatrix.h" 29cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/LiveStackAnalysis.h" 304eed756153b84c211114a3e9186bf0cb55d4b394Benjamin Kramer#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 31f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen#include "llvm/CodeGen/MachineDominators.h" 32cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/MachineFunctionPass.h" 33cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/MachineLoopInfo.h" 34cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/MachineRegisterInfo.h" 35de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#include "llvm/CodeGen/Passes.h" 36cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/CodeGen/RegAllocRegistry.h" 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/CodeGen/RegisterClassInfo.h" 38d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/CodeGen/VirtRegMap.h" 39dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/LLVMContext.h" 40d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/PassAnalysisSupport.h" 41dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/BranchProbability.h" 4200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen#include "llvm/Support/CommandLine.h" 43cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/Support/Debug.h" 44cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen#include "llvm/Support/ErrorHandling.h" 45533f58ecdd8a4732c2f0e149387c4d8d8d4142deJakob Stoklund Olesen#include "llvm/Support/Timer.h" 46d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/raw_ostream.h" 47de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#include "llvm/Target/TargetInstrInfo.h" 48c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Target/TargetSubtargetInfo.h" 4998d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen#include <queue> 5098d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen 51cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenusing namespace llvm; 52cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 53dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "regalloc" 54dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 550db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund OlesenSTATISTIC(NumGlobalSplits, "Number of split global live ranges"); 560db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund OlesenSTATISTIC(NumLocalSplits, "Number of split local live ranges"); 570db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund OlesenSTATISTIC(NumEvicted, "Number of interferences evicted"); 580db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund Olesen 59de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic cl::opt<SplitEditor::ComplementSpillMode> SplitSpillMode( 60de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "split-spill-mode", cl::Hidden, 61de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cl::desc("Spill mode for splitting live ranges"), 62de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"), 63de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar clEnumValN(SplitEditor::SM_Size, "size", "Optimize for size"), 64de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed"), 65de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar clEnumValEnd), 66de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cl::init(SplitEditor::SM_Speed)); 67708d06f7fb5dfd9c8559aea07b042a88c65645f8Jakob Stoklund Olesen 6836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<unsigned> 6936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesLastChanceRecoloringMaxDepth("lcr-max-depth", cl::Hidden, 7036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Last chance recoloring max depth"), 7136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::init(5)); 7236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<unsigned> LastChanceRecoloringMaxInterference( 7436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "lcr-max-interf", cl::Hidden, 7536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Last chance recoloring maximum number of considered" 7636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines " interference at a time"), 7736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::init(8)); 7836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 79dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic cl::opt<bool> 80dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesExhaustiveSearch("exhaustive-register-search", cl::NotHidden, 81dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cl::desc("Exhaustive Search for registers bypassing the depth " 82dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "and interference cutoffs of last chance recoloring")); 83dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 84c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesstatic cl::opt<bool> EnableLocalReassignment( 85c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines "enable-local-reassign", cl::Hidden, 86c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines cl::desc("Local reassignment can yield better allocation decisions, but " 87c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines "may be compile time intensive"), 88c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines cl::init(false)); 89c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 90f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic cl::opt<bool> EnableDeferredSpilling( 91f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "enable-deferred-spilling", cl::Hidden, 92f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar cl::desc("Instead of spilling a variable right away, defer the actual " 93f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "code insertion to the end of the allocation. That way the " 94f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "allocator might still find a suitable coloring for this " 95f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "variable because of other evicted variables."), 96f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar cl::init(false)); 97f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// FIXME: Find a good default for this flag and remove the flag. 9936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<unsigned> 10036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesCSRFirstTimeCost("regalloc-csr-first-time-cost", 10136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Cost for first time use of callee-saved register."), 10236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::init(0), cl::Hidden); 10336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 104cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenstatic RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator", 105cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen createGreedyRegisterAllocator); 106cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 107cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesennamespace { 10892a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesenclass RAGreedy : public MachineFunctionPass, 10992a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen public RegAllocBase, 11092a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen private LiveRangeEdit::Delegate { 11136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Convenient shortcuts. 11236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef std::priority_queue<std::pair<unsigned, unsigned> > PQueue; 11336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef SmallPtrSet<LiveInterval *, 4> SmallLISet; 11436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef SmallSet<unsigned, 16> SmallVirtRegSet; 11592a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen 116cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen // context 117cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen MachineFunction *MF; 118cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 11936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Shortcuts to some useful interface. 12036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetInstrInfo *TII; 12136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetRegisterInfo *TRI; 12236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RegisterClassInfo RCI; 12336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 124cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen // analyses 125b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen SlotIndexes *Indexes; 1264eed756153b84c211114a3e9186bf0cb55d4b394Benjamin Kramer MachineBlockFrequencyInfo *MBFI; 127f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen MachineDominatorTree *DomTree; 128d0bb5e2ca05d9c942223bf05e5940bb0c6cc9d3fJakob Stoklund Olesen MachineLoopInfo *Loops; 129b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen EdgeBundles *Bundles; 130b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen SpillPlacement *SpillPlacer; 131f42b66169d75301346e3685fd2b3e45e47806367Jakob Stoklund Olesen LiveDebugVariables *DebugVars; 132de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AliasAnalysis *AA; 133f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen 134cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen // state 13536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::unique_ptr<Spiller> SpillerInstance; 13636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PQueue Queue; 1371a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen unsigned NextCascade; 13822a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 13922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // Live ranges pass through a number of stages as we try to allocate them. 14022a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // Some of the stages may also create new live ranges: 14122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // 14222a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // - Region splitting. 14322a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // - Per-block splitting. 14422a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // - Local splitting. 14522a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // - Spilling. 14622a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // 14722a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // Ranges produced by one of the stages skip the previous stages when they are 14822a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // dequeued. This improves performance because we can skip interference checks 14922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // that are unlikely to give any results. It also guarantees that the live 15022a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // range splitting algorithm terminates, something that is otherwise hard to 15122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen // ensure. 15222a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen enum LiveRangeStage { 153fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// Newly created live range that has never been queued. 154fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen RS_New, 155fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen 156fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// Only attempt assignment and eviction. Then requeue as RS_Split. 157fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen RS_Assign, 158fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen 159fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// Attempt live range splitting if assignment is impossible. 160fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen RS_Split, 161fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen 16249743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen /// Attempt more aggressive live range splitting that is guaranteed to make 16349743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen /// progress. This is used for split products that may not be making 16449743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen /// progress. 16549743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen RS_Split2, 16649743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen 167fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// Live range will be spilled. No more splitting will be attempted. 168fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen RS_Spill, 169fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen 170f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 171f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar /// Live range is in memory. Because of other evictions, it might get moved 172f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar /// in a register in the end. 173f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RS_Memory, 174f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 175fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// There is nothing more we can do to this live range. Abort compilation 176fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen /// if it can't be assigned. 177fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen RS_Done 17822a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen }; 17922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Enum CutOffStage to keep a track whether the register allocation failed 181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // because of the cutoffs encountered in last chance recoloring. 182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Note: This is used as bitmask. New value should be next power of 2. 183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines enum CutOffStage { 184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // No cutoffs encountered 185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CO_None = 0, 186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // lcr-max-depth cutoff encountered 188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CO_Depth = 1, 189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // lcr-max-interf cutoff encountered 191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CO_Interf = 2 192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint8_t CutOffInfo; 195dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 196ae43dac30037395cce2b54af0a02500985813183Eli Friedman#ifndef NDEBUG 197b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen static const char *const StageName[]; 198ae43dac30037395cce2b54af0a02500985813183Eli Friedman#endif 199b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen 2001a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // RegInfo - Keep additional information about each live range. 2011a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen struct RegInfo { 2021a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen LiveRangeStage Stage; 2031a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen 2041a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // Cascade - Eviction loop prevention. See canEvictInterference(). 2051a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen unsigned Cascade; 2061a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen 2071a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen RegInfo() : Stage(RS_New), Cascade(0) {} 2081a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen }; 2091a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen 2101a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen IndexedMap<RegInfo, VirtReg2IndexFunctor> ExtraRegInfo; 21122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 21222a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen LiveRangeStage getStage(const LiveInterval &VirtReg) const { 2131a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen return ExtraRegInfo[VirtReg.reg].Stage; 2141a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen } 2151a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen 2161a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen void setStage(const LiveInterval &VirtReg, LiveRangeStage Stage) { 2171a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 2181a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo[VirtReg.reg].Stage = Stage; 21922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen } 22022a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 22122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen template<typename Iterator> 22222a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) { 2231a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 224f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen for (;Begin != End; ++Begin) { 2251feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey unsigned Reg = *Begin; 2261a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen if (ExtraRegInfo[Reg].Stage == RS_New) 2271a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo[Reg].Stage = NewStage; 228f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen } 22922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen } 230cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 23151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen /// Cost of evicting interference. 23251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen struct EvictionCost { 23351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen unsigned BrokenHints; ///< Total number of broken hints. 23451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen float MaxWeight; ///< Maximum spill weight evicted. 23551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 23636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EvictionCost(): BrokenHints(0), MaxWeight(0) {} 23751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 2386ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick bool isMax() const { return BrokenHints == ~0u; } 2396ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick 24036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setMax() { BrokenHints = ~0u; } 24136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 24236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setBrokenHints(unsigned NHints) { BrokenHints = NHints; } 24336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 24451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen bool operator<(const EvictionCost &O) const { 24536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return std::tie(BrokenHints, MaxWeight) < 24636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::tie(O.BrokenHints, O.MaxWeight); 24751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 24851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen }; 24951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 250b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen // splitting state. 25136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::unique_ptr<SplitAnalysis> SA; 25236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::unique_ptr<SplitEditor> SE; 253b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 254eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen /// Cached per-block interference maps 255eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen InterferenceCache IntfCache; 256eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen 2577b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen /// All basic blocks where the current register has uses. 25896dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen SmallVector<SpillPlacement::BlockConstraint, 8> SplitConstraints; 259b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 26096dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen /// Global live range splitting candidate info. 26196dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen struct GlobalSplitCandidate { 26200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Register intended for assignment, or 0. 26396dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen unsigned PhysReg; 26400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 26500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // SplitKit interval index for this candidate. 26600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned IntvIdx; 26700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 26800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Interference for PhysReg. 269c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen InterferenceCache::Cursor Intf; 27000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 27100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Bundles where this candidate should be live. 27296dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen BitVector LiveBundles; 2735db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen SmallVector<unsigned, 8> ActiveBlocks; 2745db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen 275c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen void reset(InterferenceCache &Cache, unsigned Reg) { 2765db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen PhysReg = Reg; 27700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntvIdx = 0; 278c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen Intf.setPhysReg(Cache, Reg); 2795db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen LiveBundles.clear(); 2805db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen ActiveBlocks.clear(); 2815db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen } 28200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 28300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Set B[i] = C for every live bundle where B[i] was NoCand. 28400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned getBundles(SmallVectorImpl<unsigned> &B, unsigned C) { 28500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned Count = 0; 28600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen for (int i = LiveBundles.find_first(); i >= 0; 28700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen i = LiveBundles.find_next(i)) 28800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (B[i] == NoCand) { 28900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen B[i] = C; 29000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Count++; 29100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 29200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen return Count; 29300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 29496dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen }; 29596dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 29636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Candidate info for each PhysReg in AllocationOrder. 29796dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen /// This vector never shrinks, but grows to the size of the largest register 29896dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen /// class. 29996dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen SmallVector<GlobalSplitCandidate, 32> GlobalCand; 30096dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 30136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines enum : unsigned { NoCand = ~0u }; 30200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 30300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen /// Candidate map. Each edge bundle is assigned to a GlobalCand entry, or to 30400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen /// NoCand which indicates the stack interval. 30500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SmallVector<unsigned, 32> BundleCand; 30600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /// Callee-save register cost, calculated once per machine function. 308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BlockFrequency CSRCost; 309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 310c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// Run or not the local reassignment heuristic. This information is 311c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// obtained from the TargetSubtargetInfo. 312c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool EnableLocalReassign; 313c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 314ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Set of broken hints that may be reconciled later because of eviction. 315ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallSetVector<LiveInterval *, 8> SetOfBrokenHints; 316ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 317cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenpublic: 318cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen RAGreedy(); 319cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 320cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen /// Return the pass name. 32136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const char* getPassName() const override { 322533f58ecdd8a4732c2f0e149387c4d8d8d4142deJakob Stoklund Olesen return "Greedy Register Allocator"; 323cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen } 324cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 325cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen /// RAGreedy analysis usage. 32636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void getAnalysisUsage(AnalysisUsage &AU) const override; 32736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void releaseMemory() override; 32836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Spiller &spiller() override { return *SpillerInstance; } 32936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void enqueue(LiveInterval *LI) override; 33036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveInterval *dequeue() override; 33136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned selectOrSplit(LiveInterval&, SmallVectorImpl<unsigned>&) override; 332ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines void aboutToRemoveInterval(LiveInterval &) override; 333cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 334cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen /// Perform register allocation. 33536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnMachineFunction(MachineFunction &mf) override; 336cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 337cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen static char ID; 338b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trick 339b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trickprivate: 34036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned selectOrSplitImpl(LiveInterval &, SmallVectorImpl<unsigned> &, 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &, unsigned = 0); 34236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 34336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool LRE_CanEraseVirtReg(unsigned) override; 34436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void LRE_WillShrinkVirtReg(unsigned) override; 34536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void LRE_DidCloneVirtReg(unsigned, unsigned) override; 34636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void enqueue(PQueue &CurQueue, LiveInterval *LI); 34736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveInterval *dequeue(PQueue &CurQueue); 34892a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen 34903ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency calcSpillCost(); 35003ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen bool addSplitConstraints(InterferenceCache::Cursor, BlockFrequency&); 351f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen void addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>); 352c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen void growRegion(GlobalSplitCandidate &Cand); 35303ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency calcGlobalSplitCost(GlobalSplitCandidate&); 35487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen bool calcCompactRegion(GlobalSplitCandidate&); 35500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>); 356034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen void calcGapWeights(unsigned, SmallVectorImpl<float>&); 3578adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick unsigned canReassign(LiveInterval &VirtReg, unsigned PhysReg); 35851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool); 35951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&); 36051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen void evictInterference(LiveInterval&, unsigned, 3611feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 36236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg, 36336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallLISet &RecoloringCandidates, 36436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SmallVirtRegSet &FixedRegisters); 365b64d92e29f38002e52a22fe36ea2d488968e3537Jakob Stoklund Olesen 3666bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen unsigned tryAssign(LiveInterval&, AllocationOrder&, 3671feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 36898c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen unsigned tryEvict(LiveInterval&, AllocationOrder&, 3691feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&, unsigned = ~0u); 370b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen unsigned tryRegionSplit(LiveInterval&, AllocationOrder&, 3711feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 37236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Calculate cost of region splitting. 37336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned calculateRegionSplitCost(LiveInterval &VirtReg, 37436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AllocationOrder &Order, 37536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BlockFrequency &BestCost, 37636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &NumCands, bool IgnoreCSR); 37736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Perform region splitting. 37836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned doRegionSplit(LiveInterval &VirtReg, unsigned BestCand, 37936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool HasCompact, 38036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs); 38136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Check other options before using a callee-saved register for the first 38236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// time. 38336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order, 38436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned PhysReg, unsigned &CostPerUseLimit, 38536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs); 386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void initializeCSRCost(); 387dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen unsigned tryBlockSplit(LiveInterval&, AllocationOrder&, 3881feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 389d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen unsigned tryInstructionSplit(LiveInterval&, AllocationOrder&, 3901feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 391034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen unsigned tryLocalSplit(LiveInterval&, AllocationOrder&, 3921feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 393b64d92e29f38002e52a22fe36ea2d488968e3537Jakob Stoklund Olesen unsigned trySplit(LiveInterval&, AllocationOrder&, 3941feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&); 39536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned tryLastChanceRecoloring(LiveInterval &, AllocationOrder &, 39636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &, 39736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &, unsigned); 39836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool tryRecoloringCandidates(PQueue &, SmallVectorImpl<unsigned> &, 39936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &, unsigned); 400ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines void tryHintRecoloring(LiveInterval &); 401ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines void tryHintsRecoloring(); 402ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 403ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Model the information carried by one end of a copy. 404ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines struct HintInfo { 405ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// The frequency of the copy. 406ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BlockFrequency Freq; 407ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// The virtual register or physical register. 408ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned Reg; 409ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Its currently assigned register. 410ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// In case of a physical register Reg == PhysReg. 411ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned PhysReg; 412ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines HintInfo(BlockFrequency Freq, unsigned Reg, unsigned PhysReg) 413ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : Freq(Freq), Reg(Reg), PhysReg(PhysReg) {} 414ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines }; 415ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines typedef SmallVector<HintInfo, 4> HintsInfo; 416ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BlockFrequency getBrokenHintFreq(const HintsInfo &, unsigned); 417ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines void collectHintInfo(unsigned, HintsInfo &); 418f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 419f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool isUnusedCalleeSavedReg(unsigned PhysReg) const; 420cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen}; 421cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} // end anonymous namespace 422cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 423cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenchar RAGreedy::ID = 0; 424cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 425b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen#ifndef NDEBUG 426b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesenconst char *const RAGreedy::StageName[] = { 427fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen "RS_New", 428fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen "RS_Assign", 429fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen "RS_Split", 43049743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen "RS_Split2", 431fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen "RS_Spill", 432f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "RS_Memory", 433fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen "RS_Done" 434b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen}; 435b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen#endif 436b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen 437200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen// Hysteresis to use when comparing floats. 438200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen// This helps stabilize decisions based on float comparisons. 43936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesconst float Hysteresis = (2007 / 2048.0f); // 0.97998046875 440200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen 441200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen 442cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund OlesenFunctionPass* llvm::createGreedyRegisterAllocator() { 443cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen return new RAGreedy(); 444cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 445cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 4461a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund OlesenRAGreedy::RAGreedy(): MachineFunctionPass(ID) { 447cfafc54040cc9722995558124f253d05a038176bJakob Stoklund Olesen initializeLiveDebugVariablesPass(*PassRegistry::getPassRegistry()); 448b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); 449cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); 450cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); 4515b220213bfe9c37c2bb41a7ae0804e06a14f1007Rafael Espindola initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); 45242b7a71dc7381d1f38bf7b7201fc26dd80453364Andrew Trick initializeMachineSchedulerPass(*PassRegistry::getPassRegistry()); 453cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeLiveStacksPass(*PassRegistry::getPassRegistry()); 454cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry()); 455cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); 456cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen initializeVirtRegMapPass(*PassRegistry::getPassRegistry()); 457042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen initializeLiveRegMatrixPass(*PassRegistry::getPassRegistry()); 458b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen initializeEdgeBundlesPass(*PassRegistry::getPassRegistry()); 459b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen initializeSpillPlacementPass(*PassRegistry::getPassRegistry()); 460cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 461cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 462cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenvoid RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const { 463cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.setPreservesCFG(); 4644eed756153b84c211114a3e9186bf0cb55d4b394Benjamin Kramer AU.addRequired<MachineBlockFrequencyInfo>(); 4654eed756153b84c211114a3e9186bf0cb55d4b394Benjamin Kramer AU.addPreserved<MachineBlockFrequencyInfo>(); 466f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AU.addRequired<AAResultsWrapperPass>(); 467f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AU.addPreserved<AAResultsWrapperPass>(); 468cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addRequired<LiveIntervals>(); 46905ec712e7f75635abbdd84dced69f4a45fe0f541Jakob Stoklund Olesen AU.addPreserved<LiveIntervals>(); 470b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen AU.addRequired<SlotIndexes>(); 471cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addPreserved<SlotIndexes>(); 472cfafc54040cc9722995558124f253d05a038176bJakob Stoklund Olesen AU.addRequired<LiveDebugVariables>(); 473cfafc54040cc9722995558124f253d05a038176bJakob Stoklund Olesen AU.addPreserved<LiveDebugVariables>(); 474cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addRequired<LiveStacks>(); 475cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addPreserved<LiveStacks>(); 476f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen AU.addRequired<MachineDominatorTree>(); 477f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen AU.addPreserved<MachineDominatorTree>(); 478cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addRequired<MachineLoopInfo>(); 479cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addPreserved<MachineLoopInfo>(); 480cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addRequired<VirtRegMap>(); 481cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen AU.addPreserved<VirtRegMap>(); 482042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen AU.addRequired<LiveRegMatrix>(); 483042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen AU.addPreserved<LiveRegMatrix>(); 484b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen AU.addRequired<EdgeBundles>(); 485b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen AU.addRequired<SpillPlacement>(); 486cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen MachineFunctionPass::getAnalysisUsage(AU); 487cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 488cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 48992a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen 49092a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 49192a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen// LiveRangeEdit delegate methods 49292a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 49392a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen 4947792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesenbool RAGreedy::LRE_CanEraseVirtReg(unsigned VirtReg) { 495042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (VRM->hasPhys(VirtReg)) { 496ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LiveInterval &LI = LIS->getInterval(VirtReg); 497ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Matrix->unassign(LI); 498ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines aboutToRemoveInterval(LI); 4997792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen return true; 5007792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen } 5017792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen // Unassigned virtreg is probably in the priority queue. 5027792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen // RegAllocBase will erase it after dequeueing. 5037792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen return false; 5047792e980c43536814ea42448db9799b4da32fef6Jakob Stoklund Olesen} 50592a55f4bdd120cdd3bb5a004c792d4d24a940311Jakob Stoklund Olesen 5061d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesenvoid RAGreedy::LRE_WillShrinkVirtReg(unsigned VirtReg) { 507042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (!VRM->hasPhys(VirtReg)) 5081d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen return; 5091d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen 5101d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen // Register is assigned, put it back on the queue for reassignment. 5111d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen LiveInterval &LI = LIS->getInterval(VirtReg); 512042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen Matrix->unassign(LI); 5131d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen enqueue(&LI); 5141d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen} 5151d5b84508173b93faf513032b3847152e6060791Jakob Stoklund Olesen 516f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesenvoid RAGreedy::LRE_DidCloneVirtReg(unsigned New, unsigned Old) { 5170d4fea786670473b53d285be378e619399e03488Jakob Stoklund Olesen // Cloning a register we haven't even heard about yet? Just ignore it. 5180d4fea786670473b53d285be378e619399e03488Jakob Stoklund Olesen if (!ExtraRegInfo.inBounds(Old)) 5190d4fea786670473b53d285be378e619399e03488Jakob Stoklund Olesen return; 5200d4fea786670473b53d285be378e619399e03488Jakob Stoklund Olesen 521f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen // LRE may clone a virtual register because dead code elimination causes it to 522165e231c4295deb5cabd124d08e231b551bcc0b2Jakob Stoklund Olesen // be split into connected components. The new components are much smaller 523165e231c4295deb5cabd124d08e231b551bcc0b2Jakob Stoklund Olesen // than the original, so they should get a new chance at being assigned. 524f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen // same stage as the parent. 525165e231c4295deb5cabd124d08e231b551bcc0b2Jakob Stoklund Olesen ExtraRegInfo[Old].Stage = RS_Assign; 5261a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.grow(New); 5271a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo[New] = ExtraRegInfo[Old]; 528f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen} 529f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen 530cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenvoid RAGreedy::releaseMemory() { 53137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SpillerInstance.reset(); 5321a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.clear(); 5335db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen GlobalCand.clear(); 534cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 535cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 53636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid RAGreedy::enqueue(LiveInterval *LI) { enqueue(Queue, LI); } 53736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 53836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) { 53998d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen // Prioritize live ranges by size, assigning larger ranges first. 54098d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen // The queue holds (size, reg) pairs. 541107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen const unsigned Size = LI->getSize(); 542107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen const unsigned Reg = LI->reg; 54398d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen assert(TargetRegisterInfo::isVirtualRegister(Reg) && 54498d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen "Can only enqueue virtual registers"); 545107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen unsigned Prio; 54690c1d7ddfc65654f7efe72d56cad65d1af9e6b2aJakob Stoklund Olesen 5471a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.grow(Reg); 5481a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen if (ExtraRegInfo[Reg].Stage == RS_New) 549fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen ExtraRegInfo[Reg].Stage = RS_Assign; 550f22ca3fe5f0cfbb832cf41270f97cf5c0134fd7bJakob Stoklund Olesen 551cc07e04262fe4bc35469fbadc53d2ec7bfd02fe2Jakob Stoklund Olesen if (ExtraRegInfo[Reg].Stage == RS_Split) { 552eb29157d80847c207b77910bcd40a6a6c91ca5c5Jakob Stoklund Olesen // Unsplit ranges that couldn't be allocated immediately are deferred until 553a16a25ddeaf495b78b04e2a19feeac00d9824e63Jakob Stoklund Olesen // everything else has been allocated. 554a16a25ddeaf495b78b04e2a19feeac00d9824e63Jakob Stoklund Olesen Prio = Size; 555f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (ExtraRegInfo[Reg].Stage == RS_Memory) { 556f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Memory operand should be considered last. 557f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Change the priority such that Memory operand are assigned in 558f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the reverse order that they came in. 559f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: Make this a member variable and probably do something about hints. 560f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar static unsigned MemOp = 0; 561f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Prio = MemOp++; 562cc07e04262fe4bc35469fbadc53d2ec7bfd02fe2Jakob Stoklund Olesen } else { 56336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Giant live ranges fall back to the global assignment heuristic, which 56436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // prevents excessive spilling in pathological cases. 56536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool ReverseLocal = TRI->reverseLocalAssignment(); 5660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar const TargetRegisterClass &RC = *MRI->getRegClass(Reg); 56737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool ForceGlobal = !ReverseLocal && 5680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (Size / SlotIndex::InstrDist) > (2 * RC.getNumRegs()); 56936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 57036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ExtraRegInfo[Reg].Stage == RS_Assign && !ForceGlobal && !LI->empty() && 5716ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick LIS->intervalIsInOneMBB(*LI)) { 5726ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // Allocate original local ranges in linear instruction order. Since they 5736ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // are singly defined, this produces optimal coloring in the absence of 5746ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // global interference and other constraints. 57536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ReverseLocal) 57636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Prio = LI->beginIndex().getInstrDistance(Indexes->getLastIndex()); 57736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else { 57836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Allocating bottom up may allow many short LRGs to be assigned first 57936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to one of the cheap registers. This could be much faster for very 58036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // large blocks on targets with many physical registers. 5810c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Prio = Indexes->getZeroIndex().getInstrDistance(LI->endIndex()); 58236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 5830c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Prio |= RC.AllocationPriority << 24; 5840c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } else { 5856ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // Allocate global and split ranges in long->short order. Long ranges that 5866ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // don't fit should be spilled (or split) ASAP so they don't create 5876ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // interference. Mark a bit to prioritize global above local ranges. 5886ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick Prio = (1u << 29) + Size; 5896ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick } 5906ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // Mark a higher bit to prioritize global and local above RS_Split. 5916ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick Prio |= (1u << 31); 592d2a50734234a80893ad71da90d9f32032c47e000Jakob Stoklund Olesen 593eb29157d80847c207b77910bcd40a6a6c91ca5c5Jakob Stoklund Olesen // Boost ranges that have a physical register hint. 594fc6374439edf2f74da4026f4cea8e341d092be5cJakob Stoklund Olesen if (VRM->hasKnownPreference(Reg)) 595eb29157d80847c207b77910bcd40a6a6c91ca5c5Jakob Stoklund Olesen Prio |= (1u << 30); 596eb29157d80847c207b77910bcd40a6a6c91ca5c5Jakob Stoklund Olesen } 597bef4c3e069a66c8b2d5871468cb57978f44ddc54Andrew Trick // The virtual register number is a tie breaker for same-sized ranges. 598bef4c3e069a66c8b2d5871468cb57978f44ddc54Andrew Trick // Give lower vreg numbers higher priority to assign them first. 59936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurQueue.push(std::make_pair(Prio, ~Reg)); 60090c1d7ddfc65654f7efe72d56cad65d1af9e6b2aJakob Stoklund Olesen} 60190c1d7ddfc65654f7efe72d56cad65d1af9e6b2aJakob Stoklund Olesen 60236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesLiveInterval *RAGreedy::dequeue() { return dequeue(Queue); } 60336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 60436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesLiveInterval *RAGreedy::dequeue(PQueue &CurQueue) { 60536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CurQueue.empty()) 606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 60736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveInterval *LI = &LIS->getInterval(~CurQueue.top().second); 60836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurQueue.pop(); 60998d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen return LI; 61098d9648de7d571b2e6d139b65961a70d1833b0d7Jakob Stoklund Olesen} 611770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen 6126bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 6136bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 6146bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen// Direct Assignment 6156bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 6166bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 6176bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen/// tryAssign - Try to assign VirtReg to an available register. 6186bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesenunsigned RAGreedy::tryAssign(LiveInterval &VirtReg, 6196bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen AllocationOrder &Order, 6201feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 6216bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen Order.rewind(); 6226bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen unsigned PhysReg; 623042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen while ((PhysReg = Order.next())) 624042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (!Matrix->checkInterference(VirtReg, PhysReg)) 6256bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen break; 626f7999fe1cb2c2bdb0a4080efabb4743719ce45caJakob Stoklund Olesen if (!PhysReg || Order.isHint()) 6276bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen return PhysReg; 6286bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 62951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // PhysReg is available, but there may be a better choice. 63051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 63151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // If we missed a simple hint, try to cheaply evict interference from the 63251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // preferred register. 63351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (unsigned Hint = MRI->getSimpleHint(VirtReg.reg)) 634042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Order.isHint(Hint)) { 63551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen DEBUG(dbgs() << "missed hint " << PrintReg(Hint, TRI) << '\n'); 63636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EvictionCost MaxCost; 63736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaxCost.setBrokenHints(1); 63851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (canEvictInterference(VirtReg, Hint, true, MaxCost)) { 63951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen evictInterference(VirtReg, Hint, NewVRegs); 64051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen return Hint; 64151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 64251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 64351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 64451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Try to evict interference from a cheaper alternative. 6456bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen unsigned Cost = TRI->getCostPerUse(PhysReg); 6466bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 6476bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen // Most registers have 0 additional cost. 6486bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen if (!Cost) 6496bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen return PhysReg; 6506bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 6516bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is available at cost " << Cost 6526bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen << '\n'); 6536bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen unsigned CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost); 6546bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen return CheapReg ? CheapReg : PhysReg; 6556bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen} 6566bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 6576bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen 658770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen//===----------------------------------------------------------------------===// 65998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen// Interference eviction 66098c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen//===----------------------------------------------------------------------===// 6612710638db2eb84cd7eefb8bb9a1b7e5c49413d45Jakob Stoklund Olesen 6628adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trickunsigned RAGreedy::canReassign(LiveInterval &VirtReg, unsigned PrevReg) { 663f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix); 6648adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick unsigned PhysReg; 6658adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick while ((PhysReg = Order.next())) { 6668adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick if (PhysReg == PrevReg) 6678adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick continue; 6688adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick 6698adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick MCRegUnitIterator Units(PhysReg, TRI); 6708adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick for (; Units.isValid(); ++Units) { 6718adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick // Instantiate a "subquery", not to be confused with the Queries array. 6728adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick LiveIntervalUnion::Query subQ(&VirtReg, &Matrix->getLiveUnions()[*Units]); 6738adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick if (subQ.checkInterference()) 6748adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick break; 6758adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick } 6768adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick // If no units have interference, break out with the current PhysReg. 6778adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick if (!Units.isValid()) 6788adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick break; 6798adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick } 6808adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick if (PhysReg) 6818adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick DEBUG(dbgs() << "can reassign: " << VirtReg << " from " 6828adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick << PrintReg(PrevReg, TRI) << " to " << PrintReg(PhysReg, TRI) 6838adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick << '\n'); 6848adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick return PhysReg; 6858adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick} 6868adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick 68751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// shouldEvict - determine if A should evict the assigned live range B. The 68851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// eviction policy defined by this function together with the allocation order 68951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// defined by enqueue() decides which registers ultimately end up being split 69051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// and spilled. 691b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen/// 6921a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen/// Cascade numbers are used to prevent infinite loops if this function is a 6931a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen/// cyclic relation. 69451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// 69551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param A The live range to be assigned. 69651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param IsHint True when A is about to be assigned to its preferred 69751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// register. 69851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param B The live range to be evicted. 69951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param BreaksHint True when B is already assigned to its preferred register. 70051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesenbool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint, 70151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen LiveInterval &B, bool BreaksHint) { 70249743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen bool CanSplit = getStage(B) < RS_Spill; 70351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 70451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Be fairly aggressive about following hints as long as the evictee can be 70551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // split. 70651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (CanSplit && IsHint && !BreaksHint) 70751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen return true; 70851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 70936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (A.weight > B.weight) { 71036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight << '\n'); 71136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 71236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 71336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 714b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen} 715b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen 71651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// canEvictInterference - Return true if all interferences between VirtReg and 71736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// PhysReg can be evicted. 71851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// 71951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param VirtReg Live range that is about to be assigned. 72051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param PhysReg Desired register for assignment. 72167c8978617c3bce9d07210f93f6c64c715f77695Dmitri Gribenko/// @param IsHint True when PhysReg is VirtReg's preferred register. 72251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @param MaxCost Only look for cheaper candidates and update with new cost 72351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// when returning true. 72451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// @returns True when interference can be evicted cheaper than MaxCost. 72598c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesenbool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg, 72651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen bool IsHint, EvictionCost &MaxCost) { 727042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // It is only possible to evict virtual register interference. 728042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg) 729042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen return false; 730042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 7316ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick bool IsLocal = LIS->intervalIsInOneMBB(VirtReg); 7326ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick 7331a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // Find VirtReg's cascade number. This will be unassigned if VirtReg was never 7341a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // involved in an eviction before. If a cascade number was assigned, deny 7351a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // evicting anything with the same or a newer cascade number. This prevents 7361a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // infinite eviction loops. 7371a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // 7381a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // This works out so a register without a cascade number is allowed to evict 7391a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen // anything, and it can be evicted by anything. 7401a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen unsigned Cascade = ExtraRegInfo[VirtReg.reg].Cascade; 7411a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen if (!Cascade) 7421a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen Cascade = NextCascade; 7431a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen 74451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen EvictionCost Cost; 745042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { 746042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); 7473f5bedf5cbde2cc2badc86b1a0b377f6efcde71cJakob Stoklund Olesen // If there is 10 or more interferences, chances are one is heavier. 74851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (Q.collectInterferingVRegs(10) >= 10) 74998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen return false; 75098c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 7513f5bedf5cbde2cc2badc86b1a0b377f6efcde71cJakob Stoklund Olesen // Check if any interfering live range is heavier than MaxWeight. 7523f5bedf5cbde2cc2badc86b1a0b377f6efcde71cJakob Stoklund Olesen for (unsigned i = Q.interferingVRegs().size(); i; --i) { 7533f5bedf5cbde2cc2badc86b1a0b377f6efcde71cJakob Stoklund Olesen LiveInterval *Intf = Q.interferingVRegs()[i - 1]; 754042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen assert(TargetRegisterInfo::isVirtualRegister(Intf->reg) && 755042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen "Only expecting virtual register interference from query"); 75651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Never evict spill products. They cannot split or spill. 757fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen if (getStage(*Intf) == RS_Done) 7581a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen return false; 75951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Once a live range becomes small enough, it is urgent that we find a 76051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // register for it. This is indicated by an infinite spill weight. These 76151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // urgent live ranges get to evict almost anything. 7629cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen // 7639cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen // Also allow urgent evictions of unspillable ranges from a strictly 7649cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen // larger allocation order. 7659cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen bool Urgent = !VirtReg.isSpillable() && 7669cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen (Intf->isSpillable() || 7679cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg)) < 7689cda1be0aa5e9e70ae493ef6944a8c202c1c70e6Jakob Stoklund Olesen RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(Intf->reg))); 76951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Only evict older cascades or live ranges without a cascade. 77051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen unsigned IntfCascade = ExtraRegInfo[Intf->reg].Cascade; 77151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (Cascade <= IntfCascade) { 77251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (!Urgent) 77351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen return false; 77451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // We permit breaking cascades for urgent evictions. It should be the 77551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // last resort, though, so make it really expensive. 77651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen Cost.BrokenHints += 10; 77751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 77851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Would this break a satisfied hint? 77951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen bool BreaksHint = VRM->hasPreferredPhys(Intf->reg); 78051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Update eviction cost. 78151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen Cost.BrokenHints += BreaksHint; 78251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight); 78351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Abort if this would be too expensive. 78451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (!(Cost < MaxCost)) 785b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen return false; 7866ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick if (Urgent) 7876ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick continue; 78836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Apply the eviction policy for non-urgent evictions. 78936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!shouldEvict(VirtReg, IsHint, *Intf, BreaksHint)) 79036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 7916ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // If !MaxCost.isMax(), then we're just looking for a cheap register. 7926ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // Evicting another local live range in this case could lead to suboptimal 7936ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick // coloring. 7948adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick if (!MaxCost.isMax() && IsLocal && LIS->intervalIsInOneMBB(*Intf) && 795c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines (!EnableLocalReassign || !canReassign(*Intf, PhysReg))) { 7966ea2b9608a38e9b53d208ff85051e8e3ed53192cAndrew Trick return false; 7978adae96fd905daa8835b6fde5e74e25d818c7471Andrew Trick } 7982710638db2eb84cd7eefb8bb9a1b7e5c49413d45Jakob Stoklund Olesen } 7992710638db2eb84cd7eefb8bb9a1b7e5c49413d45Jakob Stoklund Olesen } 80051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen MaxCost = Cost; 80198c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen return true; 80298c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen} 80398c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 80451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// evictInterference - Evict any interferring registers that prevent VirtReg 80551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// from being assigned to Physreg. This assumes that canEvictInterference 80651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen/// returned true. 80751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesenvoid RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg, 8081feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 80951458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Make sure that VirtReg has a cascade number, and assign that cascade 81051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // number to every evicted register. These live ranges than then only be 81151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // evicted by a newer cascade, preventing infinite loops. 81251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen unsigned Cascade = ExtraRegInfo[VirtReg.reg].Cascade; 81351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (!Cascade) 81451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen Cascade = ExtraRegInfo[VirtReg.reg].Cascade = NextCascade++; 81551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 81651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen DEBUG(dbgs() << "evicting " << PrintReg(PhysReg, TRI) 81751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen << " interference: Cascade " << Cascade << '\n'); 818042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 819042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // Collect all interfering virtregs first. 820042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen SmallVector<LiveInterval*, 8> Intfs; 821042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { 822042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); 82351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen assert(Q.seenAllInterferences() && "Didn't check all interfererences."); 824042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen ArrayRef<LiveInterval*> IVR = Q.interferingVRegs(); 825042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen Intfs.append(IVR.begin(), IVR.end()); 826042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen } 827042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 828042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // Evict them second. This will invalidate the queries. 829042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (unsigned i = 0, e = Intfs.size(); i != e; ++i) { 830042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen LiveInterval *Intf = Intfs[i]; 831042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // The same VirtReg may be present in multiple RegUnits. Skip duplicates. 832042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (!VRM->hasPhys(Intf->reg)) 833042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen continue; 834042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen Matrix->unassign(*Intf); 835042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen assert((ExtraRegInfo[Intf->reg].Cascade < Cascade || 836042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen VirtReg.isSpillable() < Intf->isSpillable()) && 837042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen "Cannot decrease cascade number, illegal eviction"); 838042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen ExtraRegInfo[Intf->reg].Cascade = Cascade; 839042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen ++NumEvicted; 8401feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey NewVRegs.push_back(Intf->reg); 84151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 84251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen} 84351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 844f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Returns true if the given \p PhysReg is a callee saved register and has not 845f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// been used for allocation yet. 846f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const { 847f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg); 848f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (CSR == 0) 849f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 850f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 851f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return !Matrix->isPhysRegUsed(PhysReg); 852f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 853f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 85498c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen/// tryEvict - Try to evict all interferences for a physreg. 85576395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen/// @param VirtReg Currently unassigned virtual register. 85676395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen/// @param Order Physregs to try. 85776395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen/// @return Physreg to assign VirtReg, or 0. 85898c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesenunsigned RAGreedy::tryEvict(LiveInterval &VirtReg, 85998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen AllocationOrder &Order, 8601feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs, 8616bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen unsigned CostPerUseLimit) { 86298c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen NamedRegionTimer T("Evict", TimerGroupName, TimePassesIsEnabled); 86398c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 86451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Keep track of the cheapest interference seen so far. 86536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EvictionCost BestCost; 86636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BestCost.setMax(); 86798c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen unsigned BestPhys = 0; 8686d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen unsigned OrderLimit = Order.getOrder().size(); 86998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 87051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // When we are just looking for a reduced cost per use, don't break any 87151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // hints, and only evict smaller spill weights. 87251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (CostPerUseLimit < ~0u) { 87351458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen BestCost.BrokenHints = 0; 87451458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen BestCost.MaxWeight = VirtReg.weight; 8756d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen 8766d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen // Check of any registers in RC are below CostPerUseLimit. 8776d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg); 8786d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen unsigned MinCost = RegClassInfo.getMinCost(RC); 8796d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen if (MinCost >= CostPerUseLimit) { 88037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << TRI->getRegClassName(RC) << " minimum cost = " << MinCost 8816d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen << ", no cheaper registers to be found.\n"); 8826d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen return 0; 8836d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen } 8846d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen 8856d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen // It is normal for register classes to have a long tail of registers with 8866d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen // the same cost. We don't need to look at them if they're too expensive. 8876d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen if (TRI->getCostPerUse(Order.getOrder().back()) >= CostPerUseLimit) { 8886d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen OrderLimit = RegClassInfo.getLastCostChange(RC); 8896d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen DEBUG(dbgs() << "Only trying the first " << OrderLimit << " regs.\n"); 8906d6132986d2ef14bbf9d76f5acbf2a0bace32d69Jakob Stoklund Olesen } 89151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen } 89251458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 89398c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen Order.rewind(); 89436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (unsigned PhysReg = Order.next(OrderLimit)) { 8956bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit) 8966bfba2e5af163442a1c6b11fe14aa9df9101cfd7Jakob Stoklund Olesen continue; 89751458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // The first use of a callee-saved register in a function has cost 1. 89851458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen // Don't start using a CSR when the CostPerUseLimit is low. 899f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (CostPerUseLimit == 1 && isUnusedCalleeSavedReg(PhysReg)) { 900f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " would clobber CSR " 901f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar << PrintReg(RegClassInfo.getLastCalleeSavedAlias(PhysReg), TRI) 902f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar << '\n'); 903f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar continue; 904f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 90551458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 90651458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen if (!canEvictInterference(VirtReg, PhysReg, false, BestCost)) 90798c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen continue; 90898c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 90998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen // Best so far. 91098c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen BestPhys = PhysReg; 91151458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen 91257f1e2cee06f9b57995727d786aeb1031c5376bdJakob Stoklund Olesen // Stop if the hint can be used. 913f7999fe1cb2c2bdb0a4080efabb4743719ce45caJakob Stoklund Olesen if (Order.isHint()) 91457f1e2cee06f9b57995727d786aeb1031c5376bdJakob Stoklund Olesen break; 9152710638db2eb84cd7eefb8bb9a1b7e5c49413d45Jakob Stoklund Olesen } 9162710638db2eb84cd7eefb8bb9a1b7e5c49413d45Jakob Stoklund Olesen 91798c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen if (!BestPhys) 91898c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen return 0; 91998c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen 92051458ed09e6db0e424cd528e10b879f59915abe4Jakob Stoklund Olesen evictInterference(VirtReg, BestPhys, NewVRegs); 92198c8141b6d8fcbb9bd258ebcdd4171f55c5a8e9dJakob Stoklund Olesen return BestPhys; 922b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trick} 923b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trick 924770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen 925770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen//===----------------------------------------------------------------------===// 926b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen// Region Splitting 927b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen//===----------------------------------------------------------------------===// 928b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 9291b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen/// addSplitConstraints - Fill out the SplitConstraints vector based on the 9301b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen/// interference pattern in Physreg and its aliases. Add the constraints to 9311b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen/// SpillPlacement and return the static cost of this split in Cost, assuming 9321b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen/// that all preferences in SplitConstraints are met. 933f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen/// Return false if there are no bundles with positive bias. 934f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesenbool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, 93503ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency &Cost) { 936db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); 937eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen 938b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen // Reset interference dependent info. 939db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen SplitConstraints.resize(UseBlocks.size()); 94003ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency StaticCost = 0; 941db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen for (unsigned i = 0; i != UseBlocks.size(); ++i) { 942db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; 94396dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen SpillPlacement::BlockConstraint &BC = SplitConstraints[i]; 94496dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 945f0ac26c51173a9a1d6e5b5794107dccc4c5b5792Jakob Stoklund Olesen BC.Number = BI.MBB->getNumber(); 946eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen Intf.moveToBlock(BC.Number); 947db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen BC.Entry = BI.LiveIn ? SpillPlacement::PrefReg : SpillPlacement::DontCare; 948db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen BC.Exit = BI.LiveOut ? SpillPlacement::PrefReg : SpillPlacement::DontCare; 949453f4f01302f00651aae2fc7658f6e23a2beadb0David Blaikie BC.ChangesValue = BI.FirstDef.isValid(); 950b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 951eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen if (!Intf.hasInterference()) 952eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen continue; 953eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen 95496dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen // Number of spill code instructions to insert. 95596dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen unsigned Ins = 0; 95696dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 95796dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen // Interference for the live-in value. 958eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen if (BI.LiveIn) { 959de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Intf.first() <= Indexes->getMBBStartIdx(BC.Number)) { 960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BC.Entry = SpillPlacement::MustSpill; 961de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++Ins; 962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Intf.first() < BI.FirstInstr) { 963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BC.Entry = SpillPlacement::PrefSpill; 96496dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen ++Ins; 965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Intf.first() < BI.LastInstr) { 966de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++Ins; 967de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 968a50c539b7a9e74597da34bfaea5429a48481f18bJakob Stoklund Olesen } 969b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 97096dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen // Interference for the live-out value. 971eda0fe8d58b0aaff5f18e7f13edfda3022384e70Jakob Stoklund Olesen if (BI.LiveOut) { 972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Intf.last() >= SA->getLastSplitPoint(BC.Number)) { 973de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BC.Exit = SpillPlacement::MustSpill; 97496dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen ++Ins; 975de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Intf.last() > BI.LastInstr) { 976de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BC.Exit = SpillPlacement::PrefSpill; 977de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++Ins; 978de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Intf.last() > BI.FirstInstr) { 979de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++Ins; 980de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 981b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen } 982b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 98396dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen // Accumulate the total frequency of inserted spill code. 98403ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen while (Ins--) 98503ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen StaticCost += SpillPlacer->getBlockFrequency(BC.Number); 986b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen } 987f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen Cost = StaticCost; 988db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 9891b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen // Add constraints for use-blocks. Note that these are the only constraints 9901b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen // that may add a positive bias, it is downhill from here. 9911b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen SpillPlacer->addConstraints(SplitConstraints); 992f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen return SpillPlacer->scanActiveBundles(); 993f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen} 994db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 995db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 996f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen/// addThroughConstraints - Add constraints and links to SpillPlacer from the 997f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen/// live-through blocks in Blocks. 998f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesenvoid RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf, 999f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen ArrayRef<unsigned> Blocks) { 10001b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen const unsigned GroupSize = 8; 10011b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen SpillPlacement::BlockConstraint BCS[GroupSize]; 1002f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned TBS[GroupSize]; 1003f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned B = 0, T = 0; 1004db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 1005f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen for (unsigned i = 0; i != Blocks.size(); ++i) { 1006f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned Number = Blocks[i]; 10071b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen Intf.moveToBlock(Number); 10081b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen 10097b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen if (!Intf.hasInterference()) { 1010f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen assert(T < GroupSize && "Array overflow"); 1011f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen TBS[T] = Number; 1012f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen if (++T == GroupSize) { 101339b5abf507b43da6b92f68b86406e0015ead18e9Frits van Bommel SpillPlacer->addLinks(makeArrayRef(TBS, T)); 1014f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen T = 0; 1015f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen } 10167b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen continue; 10171b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen } 10181b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen 1019f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen assert(B < GroupSize && "Array overflow"); 1020f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen BCS[B].Number = Number; 1021f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen 10227b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen // Interference for the live-in value. 10237b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen if (Intf.first() <= Indexes->getMBBStartIdx(Number)) 10247b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen BCS[B].Entry = SpillPlacement::MustSpill; 10257b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen else 10267b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen BCS[B].Entry = SpillPlacement::PrefSpill; 10277b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen 10287b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen // Interference for the live-out value. 10297b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen if (Intf.last() >= SA->getLastSplitPoint(Number)) 10307b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen BCS[B].Exit = SpillPlacement::MustSpill; 10317b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen else 10327b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen BCS[B].Exit = SpillPlacement::PrefSpill; 10337b41fbe87234f3ceef6ae11209730cbed4b69092Jakob Stoklund Olesen 10341b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen if (++B == GroupSize) { 103537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SpillPlacer->addConstraints(makeArrayRef(BCS, B)); 10361b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen B = 0; 10371b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen } 1038db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen } 1039db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 104037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SpillPlacer->addConstraints(makeArrayRef(BCS, B)); 104139b5abf507b43da6b92f68b86406e0015ead18e9Frits van Bommel SpillPlacer->addLinks(makeArrayRef(TBS, T)); 1042b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen} 1043b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 1044c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesenvoid RAGreedy::growRegion(GlobalSplitCandidate &Cand) { 10455db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen // Keep track of through blocks that have not been added to SpillPlacer. 10465db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen BitVector Todo = SA->getThroughBlocks(); 10475db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen SmallVectorImpl<unsigned> &ActiveBlocks = Cand.ActiveBlocks; 10485db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen unsigned AddedTo = 0; 1049f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen#ifndef NDEBUG 1050f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned Visited = 0; 1051f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen#endif 10525db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen 1053f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen for (;;) { 1054f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen ArrayRef<unsigned> NewBundles = SpillPlacer->getRecentPositive(); 1055f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen // Find new through blocks in the periphery of PrefRegBundles. 1056f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen for (int i = 0, e = NewBundles.size(); i != e; ++i) { 1057f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned Bundle = NewBundles[i]; 1058f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen // Look at all blocks connected to Bundle in the full graph. 1059f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen ArrayRef<unsigned> Blocks = Bundles->getBlocks(Bundle); 1060f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end(); 1061f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen I != E; ++I) { 1062f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen unsigned Block = *I; 10635db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen if (!Todo.test(Block)) 1064f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen continue; 10655db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen Todo.reset(Block); 1066f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen // This is a new through block. Add it to SpillPlacer later. 10675db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen ActiveBlocks.push_back(Block); 1068f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen#ifndef NDEBUG 1069f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen ++Visited; 1070f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen#endif 1071f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen } 1072f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen } 1073f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen // Any new blocks to add? 1074549019792a8b14500cab093ac8f3c5f7331e86d7Jakob Stoklund Olesen if (ActiveBlocks.size() == AddedTo) 1075549019792a8b14500cab093ac8f3c5f7331e86d7Jakob Stoklund Olesen break; 1076b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen 1077b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen // Compute through constraints from the interference, or assume that all 1078b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen // through blocks prefer spilling when forming compact regions. 107937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto NewBlocks = makeArrayRef(ActiveBlocks).slice(AddedTo); 1080b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen if (Cand.PhysReg) 1081b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen addThroughConstraints(Cand.Intf, NewBlocks); 1082b4666364f4db4889883d7a6c02a177ebcde7c240Jakob Stoklund Olesen else 1083b87f91b063a0ac853735f2af3bd94fb8551a11ffJakob Stoklund Olesen // Provide a strong negative bias on through blocks to prevent unwanted 1084b87f91b063a0ac853735f2af3bd94fb8551a11ffJakob Stoklund Olesen // liveness on loop backedges. 1085b87f91b063a0ac853735f2af3bd94fb8551a11ffJakob Stoklund Olesen SpillPlacer->addPrefSpill(NewBlocks, /* Strong= */ true); 1086549019792a8b14500cab093ac8f3c5f7331e86d7Jakob Stoklund Olesen AddedTo = ActiveBlocks.size(); 1087549019792a8b14500cab093ac8f3c5f7331e86d7Jakob Stoklund Olesen 1088f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen // Perhaps iterating can enable more bundles? 1089f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen SpillPlacer->iterate(); 1090f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen } 1091f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen DEBUG(dbgs() << ", v=" << Visited); 1092f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen} 109396dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 109487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// calcCompactRegion - Compute the set of edge bundles that should be live 109587972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// when splitting the current live range into compact regions. Compact 109687972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// regions can be computed without looking at interference. They are the 109787972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// regions formed by removing all the live-through blocks from the live range. 109887972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// 109987972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// Returns false if the current live range is already compact, or if the 110087972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen/// compact regions would form single block regions anyway. 110187972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesenbool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { 110287972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen // Without any through blocks, the live range is already compact. 110387972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen if (!SA->getNumThroughBlocks()) 110487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen return false; 110587972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 110687972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen // Compact regions don't correspond to any physreg. 110787972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen Cand.reset(IntfCache, 0); 110887972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 110987972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen DEBUG(dbgs() << "Compact region bundles"); 111087972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 111187972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen // Use the spill placer to determine the live bundles. GrowRegion pretends 111287972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen // that all the through blocks have interference when PhysReg is unset. 111387972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen SpillPlacer->prepare(Cand.LiveBundles); 111487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 111587972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen // The static split cost will be zero since Cand.Intf reports no interference. 111603ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency Cost; 111787972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen if (!addSplitConstraints(Cand.Intf, Cost)) { 111887972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen DEBUG(dbgs() << ", none.\n"); 111987972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen return false; 112087972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen } 112187972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 112287972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen growRegion(Cand); 112387972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen SpillPlacer->finish(); 112487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 112587972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen if (!Cand.LiveBundles.any()) { 112687972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen DEBUG(dbgs() << ", none.\n"); 112787972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen return false; 112887972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen } 112987972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 113087972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen DEBUG({ 113187972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen for (int i = Cand.LiveBundles.find_first(); i>=0; 113287972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen i = Cand.LiveBundles.find_next(i)) 113387972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen dbgs() << " EB#" << i; 113487972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen dbgs() << ".\n"; 113587972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen }); 113687972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen return true; 113787972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen} 113887972fa63f0e2631778166e0c258c456ec12db7cJakob Stoklund Olesen 1139200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen/// calcSpillCost - Compute how expensive it would be to split the live range in 1140200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen/// SA around all use blocks instead of forming bundle regions. 114103ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund OlesenBlockFrequency RAGreedy::calcSpillCost() { 114203ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency Cost = 0; 1143200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); 1144200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen for (unsigned i = 0; i != UseBlocks.size(); ++i) { 1145200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; 1146200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen unsigned Number = BI.MBB->getNumber(); 1147200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen // We normally only need one spill instruction - a load or a store. 1148200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen Cost += SpillPlacer->getBlockFrequency(Number); 1149200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen 1150200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen // Unless the value is redefined in the block. 11513f5beede1bb97ba4e06dc300e00b70e1013e7216Jakob Stoklund Olesen if (BI.LiveIn && BI.LiveOut && BI.FirstDef) 11523f5beede1bb97ba4e06dc300e00b70e1013e7216Jakob Stoklund Olesen Cost += SpillPlacer->getBlockFrequency(Number); 1153200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen } 1154200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen return Cost; 1155200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen} 1156200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen 1157b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen/// calcGlobalSplitCost - Return the global split cost of following the split 1158b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen/// pattern in LiveBundles. This cost should be added to the local cost of the 115996dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen/// interference pattern in SplitConstraints. 1160b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen/// 116103ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund OlesenBlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) { 116203ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency GlobalCost = 0; 11635db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen const BitVector &LiveBundles = Cand.LiveBundles; 1164db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); 1165db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen for (unsigned i = 0; i != UseBlocks.size(); ++i) { 1166db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; 116796dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen SpillPlacement::BlockConstraint &BC = SplitConstraints[i]; 1168874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen bool RegIn = LiveBundles[Bundles->getBundle(BC.Number, 0)]; 1169874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen bool RegOut = LiveBundles[Bundles->getBundle(BC.Number, 1)]; 1170874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen unsigned Ins = 0; 1171874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen 1172db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen if (BI.LiveIn) 1173db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen Ins += RegIn != (BC.Entry == SpillPlacement::PrefReg); 1174db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen if (BI.LiveOut) 1175db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen Ins += RegOut != (BC.Exit == SpillPlacement::PrefReg); 117603ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen while (Ins--) 117703ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen GlobalCost += SpillPlacer->getBlockFrequency(BC.Number); 1178b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen } 1179db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 11805db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) { 11815db4289e404d76664f8aabe2675a4cc2d7b0e98eJakob Stoklund Olesen unsigned Number = Cand.ActiveBlocks[i]; 1182db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen bool RegIn = LiveBundles[Bundles->getBundle(Number, 0)]; 1183db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen bool RegOut = LiveBundles[Bundles->getBundle(Number, 1)]; 11849a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen if (!RegIn && !RegOut) 11859a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen continue; 11869a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen if (RegIn && RegOut) { 11879a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen // We need double spill code if this block has interference. 1188c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen Cand.Intf.moveToBlock(Number); 118903ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen if (Cand.Intf.hasInterference()) { 119003ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen GlobalCost += SpillPlacer->getBlockFrequency(Number); 119103ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen GlobalCost += SpillPlacer->getBlockFrequency(Number); 119203ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen } 11939a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen continue; 11949a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen } 11959a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen // live-in / stack-out or stack-in live-out. 11969a54352879e5aaac2e2c37490e5cb7844550db8bJakob Stoklund Olesen GlobalCost += SpillPlacer->getBlockFrequency(Number); 1197db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen } 1198b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen return GlobalCost; 1199b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen} 1200b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 120100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// splitAroundRegion - Split the current live range around the regions 120200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// determined by BundleCand and GlobalCand. 1203ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen/// 120400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// Before calling this function, GlobalCand and BundleCand must be initialized 120500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// so each bundle is assigned to a valid candidate, or NoCand for the 120600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// stack-bound bundles. The shared SA/SE SplitAnalysis and SplitEditor 120700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// objects must be initialized for the current live range, and intervals 120800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// created for the used candidates. 1209ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen/// 121000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// @param LREdit The LiveRangeEdit object handling the current split. 121100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// @param UsedCands List of used GlobalCand entries. Every BundleCand value 121200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen/// must appear in this list. 121300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesenvoid RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, 121400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen ArrayRef<unsigned> UsedCands) { 121500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // These are the intervals created for new global ranges. We may create more 121600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // intervals for local ranges. 121700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen const unsigned NumGlobalIntvs = LREdit.size(); 121800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen DEBUG(dbgs() << "splitAroundRegion with " << NumGlobalIntvs << " globals.\n"); 121900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen assert(NumGlobalIntvs && "No global intervals configured"); 1220ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 12212d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen // Isolate even single instructions when dealing with a proper sub-class. 122269145baf36219b07a49d8ce14b4a04870e72a123Jakob Stoklund Olesen // That guarantees register class inflation for the stack interval because it 12232d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen // is all copies. 12242d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen unsigned Reg = SA->getParent().reg; 12252d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); 12262d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen 122787360f73ae205854f100ba5fb7eef7b90ac3bc27Jakob Stoklund Olesen // First handle all the blocks with uses. 1228db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); 1229db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen for (unsigned i = 0; i != UseBlocks.size(); ++i) { 1230db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; 123100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned Number = BI.MBB->getNumber(); 123200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned IntvIn = 0, IntvOut = 0; 123300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SlotIndex IntfIn, IntfOut; 123400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (BI.LiveIn) { 123500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)]; 123600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (CandIn != NoCand) { 123700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[CandIn]; 123800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntvIn = Cand.IntvIdx; 123900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.Intf.moveToBlock(Number); 124000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntfIn = Cand.Intf.first(); 124100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 124200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 124300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (BI.LiveOut) { 124400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)]; 124500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (CandOut != NoCand) { 124600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[CandOut]; 124700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntvOut = Cand.IntvIdx; 124800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.Intf.moveToBlock(Number); 124900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntfOut = Cand.Intf.last(); 125000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 125100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 1252ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1253fd5c51342a429ecab86a645282d0b36b216c0256Jakob Stoklund Olesen // Create separate intervals for isolated blocks with multiple uses. 125400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (!IntvIn && !IntvOut) { 1255fd5c51342a429ecab86a645282d0b36b216c0256Jakob Stoklund Olesen DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " isolated.\n"); 12562d6d86be84ee355223ccd20b7f87a0c9971c50c9Jakob Stoklund Olesen if (SA->shouldSplitSingleBlock(BI, SingleInstrs)) 125787360f73ae205854f100ba5fb7eef7b90ac3bc27Jakob Stoklund Olesen SE->splitSingleBlock(BI); 1258fd5c51342a429ecab86a645282d0b36b216c0256Jakob Stoklund Olesen continue; 1259fd5c51342a429ecab86a645282d0b36b216c0256Jakob Stoklund Olesen } 1260fd5c51342a429ecab86a645282d0b36b216c0256Jakob Stoklund Olesen 126100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (IntvIn && IntvOut) 126200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut); 126300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen else if (IntvIn) 126400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SE->splitRegInBlock(BI, IntvIn, IntfIn); 1265b4ddedce599183362b0f0333922c2fe0e163a129Jakob Stoklund Olesen else 126600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SE->splitRegOutBlock(BI, IntvOut, IntfOut); 1267ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen } 1268ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 126900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Handle live-through blocks. The relevant live-through blocks are stored in 127000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // the ActiveBlocks list with each candidate. We need to filter out 127100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // duplicates. 127200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen BitVector Todo = SA->getThroughBlocks(); 127300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen for (unsigned c = 0; c != UsedCands.size(); ++c) { 127400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks; 127500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { 127600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned Number = Blocks[i]; 127700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (!Todo.test(Number)) 127800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen continue; 127900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Todo.reset(Number); 128000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 128100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned IntvIn = 0, IntvOut = 0; 128200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SlotIndex IntfIn, IntfOut; 128300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 128400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)]; 128500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (CandIn != NoCand) { 128600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[CandIn]; 128700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntvIn = Cand.IntvIdx; 128800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.Intf.moveToBlock(Number); 128900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntfIn = Cand.Intf.first(); 129000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 129100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 129200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)]; 129300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (CandOut != NoCand) { 129400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[CandOut]; 129500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntvOut = Cand.IntvIdx; 129600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.Intf.moveToBlock(Number); 129700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen IntfOut = Cand.Intf.last(); 129800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 129900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (!IntvIn && !IntvOut) 130000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen continue; 130100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut); 130200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 1303db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen } 1304db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen 13050db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund Olesen ++NumGlobalSplits; 1306ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 13075928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen SmallVector<unsigned, 8> IntvMap; 13085928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen SE->finish(&IntvMap); 13091feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey DebugVars->splitRegister(Reg, LREdit.regs(), *LIS); 1310f42b66169d75301346e3685fd2b3e45e47806367Jakob Stoklund Olesen 13111a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 1312b2abfa0bf30edf292a27a06e091d03983e644c9bJakob Stoklund Olesen unsigned OrigBlocks = SA->getNumLiveBlocks(); 13135928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen 13145928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // Sort out the new intervals created by splitting. We get four kinds: 13155928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // - Remainder intervals should not be split again. 13165928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // - Candidate intervals can be assigned to Cand.PhysReg. 13175928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // - Block-local splits are candidates for local splitting. 13185928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // - DCE leftovers should go back on the queue. 13195928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen for (unsigned i = 0, e = LREdit.size(); i != e; ++i) { 13201feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey LiveInterval &Reg = LIS->getInterval(LREdit.get(i)); 13215928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen 13225928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // Ignore old intervals from DCE. 13231a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen if (getStage(Reg) != RS_New) 13245928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen continue; 13255928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen 13265928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // Remainder interval. Don't try splitting again, spill if it doesn't 13275928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen // allocate. 13285928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen if (IntvMap[i] == 0) { 1329fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen setStage(Reg, RS_Spill); 13305928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen continue; 13315928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen } 13325928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen 133300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Global intervals. Allow repeated splitting as long as the number of live 133400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // blocks is strictly decreasing. 133500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (IntvMap[i] < NumGlobalIntvs) { 13361a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen if (SA->countLiveBlocks(&Reg) >= OrigBlocks) { 13379f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks 13389f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen << " blocks as original.\n"); 13399f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen // Don't allow repeated splitting as a safe guard against looping. 134049743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen setStage(Reg, RS_Split2); 13419f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen } 13429f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen continue; 13439f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen } 13449f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen 13459f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen // Other intervals are treated as new. This includes local intervals created 13469f4b893b84d9c2b56aa2abc3c96ce1e5ccc465e5Jakob Stoklund Olesen // for blocks with multiple uses, and anything created by DCE. 13475928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen } 13485928046306d8bbe7db35707c294689f515f90e56Jakob Stoklund Olesen 1349eb29157d80847c207b77910bcd40a6a6c91ca5c5Jakob Stoklund Olesen if (VerifyEnabled) 1350ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen MF->verify(this, "After splitting live range around region"); 1351ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen} 1352ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1353b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesenunsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order, 13541feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 1355c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen unsigned NumCands = 0; 135603ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency BestCost; 135700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 135800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Check if we can split this live range around a compact region. 1359a16a25ddeaf495b78b04e2a19feeac00d9824e63Jakob Stoklund Olesen bool HasCompact = calcCompactRegion(GlobalCand.front()); 136000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (HasCompact) { 136100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Yes, keep GlobalCand[0] as the compact region candidate. 136200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen NumCands = 1; 136303ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BestCost = BlockFrequency::getMaxFrequency(); 136400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } else { 136500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // No benefit from the compact region, our fallback will be per-block 136600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // splitting. Make sure we find a solution that is cheaper than spilling. 136703ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BestCost = calcSpillCost(); 136836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Cost of isolating all blocks = "; 136936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MBFI->printBlockFreq(dbgs(), BestCost) << '\n'); 137000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 137196dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 137236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned BestCand = 137336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines calculateRegionSplitCost(VirtReg, Order, BestCost, NumCands, 137436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines false/*IgnoreCSR*/); 137536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 137636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // No solutions found, fall back to single block splitting. 137736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!HasCompact && BestCand == NoCand) 137836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 0; 137936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return doRegionSplit(VirtReg, BestCand, HasCompact, NewVRegs); 138136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 138236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesunsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg, 138436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AllocationOrder &Order, 138536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BlockFrequency &BestCost, 138636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &NumCands, 138736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IgnoreCSR) { 138836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned BestCand = NoCand; 1389b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen Order.rewind(); 1390c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen while (unsigned PhysReg = Order.next()) { 1391f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IgnoreCSR && isUnusedCalleeSavedReg(PhysReg)) 1392f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar continue; 139336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1394f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen // Discard bad candidates before we run out of interference cache cursors. 1395f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen // This will only affect register classes with a lot of registers (>32). 1396f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen if (NumCands == IntfCache.getMaxCursors()) { 1397f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen unsigned WorstCount = ~0u; 1398f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen unsigned Worst = 0; 1399f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen for (unsigned i = 0; i != NumCands; ++i) { 140000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (i == BestCand || !GlobalCand[i].PhysReg) 1401f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen continue; 1402f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen unsigned Count = GlobalCand[i].LiveBundles.count(); 1403de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Count < WorstCount) { 1404de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Worst = i; 1405de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar WorstCount = Count; 1406de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1407f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen } 1408f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen --NumCands; 1409f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen GlobalCand[Worst] = GlobalCand[NumCands]; 14107bdf0060a00f04ad03d3c6f294d8db6f4951dbc2Jakob Stoklund Olesen if (BestCand == NumCands) 14117bdf0060a00f04ad03d3c6f294d8db6f4951dbc2Jakob Stoklund Olesen BestCand = Worst; 1412f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen } 1413f1c709837bd11c5383fce3b8a026a7c8eaabba86Jakob Stoklund Olesen 1414c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen if (GlobalCand.size() <= NumCands) 1415c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen GlobalCand.resize(NumCands+1); 1416c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[NumCands]; 1417c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen Cand.reset(IntfCache, PhysReg); 141896dcd95a45968de6cb05864cf91aae33169cf179Jakob Stoklund Olesen 1419c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen SpillPlacer->prepare(Cand.LiveBundles); 142003ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BlockFrequency Cost; 1421c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen if (!addSplitConstraints(Cand.Intf, Cost)) { 1422f4afdfc501b7185d24a0ef184fe3d0c0bbe22e0cJakob Stoklund Olesen DEBUG(dbgs() << PrintReg(PhysReg, TRI) << "\tno positive bundles\n"); 14231b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen continue; 14241b400e840f58489c7950f815780cf08917cecaa8Jakob Stoklund Olesen } 142536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << PrintReg(PhysReg, TRI) << "\tstatic = "; 142636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MBFI->printBlockFreq(dbgs(), Cost)); 1427200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen if (Cost >= BestCost) { 1428200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen DEBUG({ 1429200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen if (BestCand == NoCand) 1430200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen dbgs() << " worse than no bundles\n"; 1431200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen else 1432200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen dbgs() << " worse than " 1433200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen << PrintReg(GlobalCand[BestCand].PhysReg, TRI) << '\n'; 1434200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen }); 1435b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen continue; 1436874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen } 1437c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen growRegion(Cand); 1438ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 14399efa2a263ea470caacef1c85f6ca45e32bf516d3Jakob Stoklund Olesen SpillPlacer->finish(); 14409efa2a263ea470caacef1c85f6ca45e32bf516d3Jakob Stoklund Olesen 1441ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen // No live bundles, defer to splitSingleBlocks(). 1442c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen if (!Cand.LiveBundles.any()) { 1443874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen DEBUG(dbgs() << " no bundles.\n"); 1444ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen continue; 1445874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen } 1446ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1447c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen Cost += calcGlobalSplitCost(Cand); 1448874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen DEBUG({ 144936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines dbgs() << ", total = "; MBFI->printBlockFreq(dbgs(), Cost) 145036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << " with bundles"; 1451c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen for (int i = Cand.LiveBundles.find_first(); i>=0; 1452c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen i = Cand.LiveBundles.find_next(i)) 1453874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen dbgs() << " EB#" << i; 1454874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen dbgs() << ".\n"; 1455874be74179b087be36a6e7869f3aa8b70732aca1Jakob Stoklund Olesen }); 1456200729882a47535d4c2496283d26600171531fadJakob Stoklund Olesen if (Cost < BestCost) { 1457c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen BestCand = NumCands; 145803ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen BestCost = Cost; 1459b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen } 1460c66a37df73f70ec3dbed06277763624f33ee3512Jakob Stoklund Olesen ++NumCands; 1461b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen } 146236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return BestCand; 146336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 1464ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 146536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesunsigned RAGreedy::doRegionSplit(LiveInterval &VirtReg, unsigned BestCand, 146636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool HasCompact, 146736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs) { 146836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<unsigned, 8> UsedCands; 146900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Prepare split editor. 1470de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); 1471708d06f7fb5dfd9c8559aea07b042a88c65645f8Jakob Stoklund Olesen SE->reset(LREdit, SplitSpillMode); 147200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 147300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Assign all edge bundles to the preferred candidate, or NoCand. 147400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen BundleCand.assign(Bundles->getNumBundles(), NoCand); 147500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 147600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Assign bundles for the best candidate region. 147700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (BestCand != NoCand) { 147800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand[BestCand]; 147900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (unsigned B = Cand.getBundles(BundleCand, BestCand)) { 148000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen UsedCands.push_back(BestCand); 148100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.IntvIdx = SE->openIntv(); 148200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen DEBUG(dbgs() << "Split for " << PrintReg(Cand.PhysReg, TRI) << " in " 148300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen << B << " bundles, intv " << Cand.IntvIdx << ".\n"); 148432668ea7a290ee1cb6bfe8cd677cdd4e5df05b4dChandler Carruth (void)B; 148500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 148600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 148700005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 148800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen // Assign bundles for the compact region. 148900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (HasCompact) { 149000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalSplitCandidate &Cand = GlobalCand.front(); 149100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen assert(!Cand.PhysReg && "Compact region has no physreg"); 149200005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen if (unsigned B = Cand.getBundles(BundleCand, 0)) { 149300005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen UsedCands.push_back(0); 149400005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen Cand.IntvIdx = SE->openIntv(); 149500005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen DEBUG(dbgs() << "Split for compact region in " << B << " bundles, intv " 149600005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen << Cand.IntvIdx << ".\n"); 149732668ea7a290ee1cb6bfe8cd677cdd4e5df05b4dChandler Carruth (void)B; 149800005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 149900005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen } 150000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen 150100005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen splitAroundRegion(LREdit, UsedCands); 1502b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen return 0; 1503b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen} 1504b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 1505ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1506ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen//===----------------------------------------------------------------------===// 1507dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen// Per-Block Splitting 1508dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen//===----------------------------------------------------------------------===// 1509dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen 1510dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen/// tryBlockSplit - Split a global live range around every block with uses. This 1511dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen/// creates a lot of local live ranges, that will be split by tryLocalSplit if 1512dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen/// they don't allocate. 1513dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesenunsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order, 15141feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 1515dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed"); 1516dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen unsigned Reg = VirtReg.reg; 1517dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); 1518de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); 1519708d06f7fb5dfd9c8559aea07b042a88c65645f8Jakob Stoklund Olesen SE->reset(LREdit, SplitSpillMode); 1520dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); 1521dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen for (unsigned i = 0; i != UseBlocks.size(); ++i) { 1522dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; 1523dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen if (SA->shouldSplitSingleBlock(BI, SingleInstrs)) 1524dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen SE->splitSingleBlock(BI); 1525dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen } 1526dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen // No blocks were split. 1527dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen if (LREdit.empty()) 1528dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen return 0; 1529dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen 1530dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen // We did split for some blocks. 1531a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen SmallVector<unsigned, 8> IntvMap; 1532a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen SE->finish(&IntvMap); 15331f8804263ffc5e6843d81f5c7bd9c739aa90fde5Jakob Stoklund Olesen 15341f8804263ffc5e6843d81f5c7bd9c739aa90fde5Jakob Stoklund Olesen // Tell LiveDebugVariables about the new ranges. 15351feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey DebugVars->splitRegister(Reg, LREdit.regs(), *LIS); 15361f8804263ffc5e6843d81f5c7bd9c739aa90fde5Jakob Stoklund Olesen 1537a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 1538a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen 1539a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen // Sort out the new intervals created by splitting. The remainder interval 1540a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen // goes straight to spilling, the new local ranges get to stay RS_New. 1541a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen for (unsigned i = 0, e = LREdit.size(); i != e; ++i) { 15421feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey LiveInterval &LI = LIS->getInterval(LREdit.get(i)); 1543a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen if (getStage(LI) == RS_New && IntvMap[i] == 0) 1544a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen setStage(LI, RS_Spill); 1545a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen } 1546a9c41d39d1adc92107e095aca6f851aed71b6a5fJakob Stoklund Olesen 1547dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen if (VerifyEnabled) 1548dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen MF->verify(this, "After splitting live range around basic blocks"); 1549dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen return 0; 1550dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen} 1551dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen 1552d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1553d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 1554d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen// Per-Instruction Splitting 1555d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 1556d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 155736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Get the number of allocatable registers that match the constraints of \p Reg 155836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// on \p MI and that are also in \p SuperRC. 155936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic unsigned getNumAllocatableRegsForConstraints( 156036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const MachineInstr *MI, unsigned Reg, const TargetRegisterClass *SuperRC, 156136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, 156236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const RegisterClassInfo &RCI) { 156336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(SuperRC && "Invalid register class"); 156436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 156536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetRegisterClass *ConstrainedRC = 156636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MI->getRegClassConstraintEffectForVReg(Reg, SuperRC, TII, TRI, 156736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /* ExploreBundle */ true); 156836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ConstrainedRC) 156936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 0; 157036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return RCI.getNumAllocatableRegs(ConstrainedRC); 157136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 157236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1573d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// tryInstructionSplit - Split a live range around individual instructions. 1574d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// This is normally not worthwhile since the spiller is doing essentially the 1575d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// same thing. However, when the live range is in a constrained register 1576d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// class, it may help to insert copies such that parts of the live range can 1577d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// be moved to a larger register class. 1578d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// 1579d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen/// This is similar to spilling to a larger register class. 1580d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesenunsigned 1581d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund OlesenRAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, 15821feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 158336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg); 1584d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen // There is no point to this if there are no larger sub-classes. 158536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!RegClassInfo.isProperSubClass(CurRC)) 1586d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return 0; 1587d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1588d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen // Always enable split spill mode, since we're effectively spilling to a 1589d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen // register. 1590de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); 1591d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SE->reset(LREdit, SplitEditor::SM_Size); 1592d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1593d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen ArrayRef<SlotIndex> Uses = SA->getUseSlots(); 1594d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen if (Uses.size() <= 1) 1595d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return 0; 1596d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1597d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen DEBUG(dbgs() << "Split around " << Uses.size() << " individual instrs.\n"); 1598d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 15994c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const TargetRegisterClass *SuperRC = 16004c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TRI->getLargestLegalSuperClass(CurRC, *MF); 160136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned SuperRCNumAllocatableRegs = RCI.getNumAllocatableRegs(SuperRC); 160236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Split around every non-copy instruction if this split will relax 160336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the constraints on the virtual register. 160436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, splitting just inserts uncoalescable copies that do not help 160536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the allocation. 1606d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen for (unsigned i = 0; i != Uses.size(); ++i) { 1607d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen if (const MachineInstr *MI = Indexes->getInstructionFromIndex(Uses[i])) 160836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MI->isFullCopy() || 160936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SuperRCNumAllocatableRegs == 161036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines getNumAllocatableRegsForConstraints(MI, VirtReg.reg, SuperRC, TII, 161136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TRI, RCI)) { 1612d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen DEBUG(dbgs() << " skip:\t" << Uses[i] << '\t' << *MI); 1613d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen continue; 1614d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen } 1615d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SE->openIntv(); 1616d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SlotIndex SegStart = SE->enterIntvBefore(Uses[i]); 1617d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SlotIndex SegStop = SE->leaveIntvAfter(Uses[i]); 1618d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SE->useIntv(SegStart, SegStop); 1619d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen } 1620d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1621d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen if (LREdit.empty()) { 1622d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen DEBUG(dbgs() << "All uses were copies.\n"); 1623d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return 0; 1624d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen } 1625d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1626d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SmallVector<unsigned, 8> IntvMap; 1627d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen SE->finish(&IntvMap); 16281feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS); 1629d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 1630d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1631d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen // Assign all new registers to RS_Spill. This was the last chance. 1632d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen setStage(LREdit.begin(), LREdit.end(), RS_Spill); 1633d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return 0; 1634d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen} 1635d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1636d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen 1637dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen//===----------------------------------------------------------------------===// 1638034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen// Local Splitting 1639034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 1640034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1641034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1642034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// calcGapWeights - Compute the maximum spill weight that needs to be evicted 1643034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// in order to use PhysReg between two entries in SA->UseSlots. 1644034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// 1645034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// GapWeight[i] represents the gap between UseSlots[i] and UseSlots[i+1]. 1646034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// 1647034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesenvoid RAGreedy::calcGapWeights(unsigned PhysReg, 1648034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen SmallVectorImpl<float> &GapWeight) { 1649db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen assert(SA->getUseBlocks().size() == 1 && "Not a local interval"); 1650db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front(); 1651b20b518f800293ea6b2bed04134c71293ac52403Jakob Stoklund Olesen ArrayRef<SlotIndex> Uses = SA->getUseSlots(); 1652034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen const unsigned NumGaps = Uses.size()-1; 1653034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1654034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Start and end points for the interference check. 1655fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen SlotIndex StartIdx = 1656fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen BI.LiveIn ? BI.FirstInstr.getBaseIndex() : BI.FirstInstr; 1657fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen SlotIndex StopIdx = 1658fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen BI.LiveOut ? BI.LastInstr.getBoundaryIndex() : BI.LastInstr; 1659034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1660034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen GapWeight.assign(NumGaps, 0.0f); 1661034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1662034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Add interference from each overlapping register. 1663042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { 1664042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (!Matrix->query(const_cast<LiveInterval&>(SA->getParent()), *Units) 1665042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen .checkInterference()) 1666034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen continue; 1667034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1668fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen // We know that VirtReg is a continuous interval from FirstInstr to 1669fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen // LastInstr, so we don't need InterferenceQuery. 1670034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // 1671034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Interference that overlaps an instruction is counted in both gaps 1672034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // surrounding the instruction. The exception is interference before 1673034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // StartIdx and after StopIdx. 1674034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // 1675042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen LiveIntervalUnion::SegmentIter IntI = 1676042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen Matrix->getLiveUnions()[*Units] .find(StartIdx); 1677034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen for (unsigned Gap = 0; IntI.valid() && IntI.start() < StopIdx; ++IntI) { 1678034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Skip the gaps before IntI. 1679034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen while (Uses[Gap+1].getBoundaryIndex() < IntI.start()) 1680034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (++Gap == NumGaps) 1681034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1682034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Gap == NumGaps) 1683034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1684034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1685034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Update the gaps covered by IntI. 1686034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen const float weight = IntI.value()->weight; 1687034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen for (; Gap != NumGaps; ++Gap) { 1688034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen GapWeight[Gap] = std::max(GapWeight[Gap], weight); 1689034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Uses[Gap+1].getBaseIndex() >= IntI.stop()) 1690034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1691034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1692034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Gap == NumGaps) 1693034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1694034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1695034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1696042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 1697042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // Add fixed interference. 1698042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { 16994f3b5e8c9232e43d1291aab8db5f5698d7ee0ea4Matthias Braun const LiveRange &LR = LIS->getRegUnit(*Units); 17004f3b5e8c9232e43d1291aab8db5f5698d7ee0ea4Matthias Braun LiveRange::const_iterator I = LR.find(StartIdx); 17014f3b5e8c9232e43d1291aab8db5f5698d7ee0ea4Matthias Braun LiveRange::const_iterator E = LR.end(); 1702042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 1703042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen // Same loop as above. Mark any overlapped gaps as HUGE_VALF. 1704042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (unsigned Gap = 0; I != E && I->start < StopIdx; ++I) { 1705042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen while (Uses[Gap+1].getBoundaryIndex() < I->start) 1706042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (++Gap == NumGaps) 1707042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen break; 1708042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Gap == NumGaps) 1709042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen break; 1710042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen 1711042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen for (; Gap != NumGaps; ++Gap) { 1712eb3602472026dc029beb45ccbe09bc84162ba949Aaron Ballman GapWeight[Gap] = llvm::huge_valf; 1713042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Uses[Gap+1].getBaseIndex() >= I->end) 1714042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen break; 1715042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen } 1716042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Gap == NumGaps) 1717042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen break; 1718042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen } 1719042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen } 1720034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen} 1721034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1722034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// tryLocalSplit - Try to split VirtReg into smaller intervals inside its only 1723034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// basic block. 1724034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen/// 1725034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesenunsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, 17261feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 1727db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen assert(SA->getUseBlocks().size() == 1 && "Not a local interval"); 1728db529a8a5d071610f3a8b467693bc40b073e68efJakob Stoklund Olesen const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front(); 1729034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1730034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Note that it is possible to have an interval that is live-in or live-out 1731034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // while only covering a single block - A phi-def can use undef values from 1732034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // predecessors, and the block could be a single-block loop. 1733034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // We don't bother doing anything clever about such a case, we simply assume 1734fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen // that the interval is continuous from FirstInstr to LastInstr. We should 1735fe62d92b7bbaf73e576bec0c0b11cfa6c191aa87Jakob Stoklund Olesen // make sure that we don't do anything illegal to such an interval, though. 1736034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1737b20b518f800293ea6b2bed04134c71293ac52403Jakob Stoklund Olesen ArrayRef<SlotIndex> Uses = SA->getUseSlots(); 1738034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Uses.size() <= 2) 1739034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen return 0; 1740034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen const unsigned NumGaps = Uses.size()-1; 1741034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1742034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG({ 1743034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen dbgs() << "tryLocalSplit: "; 1744034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen for (unsigned i = 0, e = Uses.size(); i != e; ++i) 1745b20b518f800293ea6b2bed04134c71293ac52403Jakob Stoklund Olesen dbgs() << ' ' << Uses[i]; 1746034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen dbgs() << '\n'; 1747034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen }); 1748034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1749a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen // If VirtReg is live across any register mask operands, compute a list of 1750a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen // gaps with register masks. 1751a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen SmallVector<unsigned, 8> RegMaskGaps; 1752042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Matrix->checkRegMaskInterference(VirtReg)) { 1753a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen // Get regmask slots for the whole block. 1754a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen ArrayRef<SlotIndex> RMS = LIS->getRegMaskSlotsInBlock(BI.MBB->getNumber()); 1755cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen DEBUG(dbgs() << RMS.size() << " regmasks in block:"); 1756a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen // Constrain to VirtReg's live range. 1757cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen unsigned ri = std::lower_bound(RMS.begin(), RMS.end(), 1758cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen Uses.front().getRegSlot()) - RMS.begin(); 1759a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen unsigned re = RMS.size(); 1760a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen for (unsigned i = 0; i != NumGaps && ri != re; ++i) { 1761cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen // Look for Uses[i] <= RMS <= Uses[i+1]. 1762cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen assert(!SlotIndex::isEarlierInstr(RMS[ri], Uses[i])); 1763cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen if (SlotIndex::isEarlierInstr(Uses[i+1], RMS[ri])) 1764a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen continue; 1765cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen // Skip a regmask on the same instruction as the last use. It doesn't 1766cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen // overlap the live range. 1767cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen if (SlotIndex::isSameInstr(Uses[i+1], RMS[ri]) && i+1 == NumGaps) 1768cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen break; 1769cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen DEBUG(dbgs() << ' ' << RMS[ri] << ':' << Uses[i] << '-' << Uses[i+1]); 1770a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen RegMaskGaps.push_back(i); 1771cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen // Advance ri to the next gap. A regmask on one of the uses counts in 1772cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen // both gaps. 1773cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen while (ri != re && SlotIndex::isEarlierInstr(RMS[ri], Uses[i+1])) 1774cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen ++ri; 1775a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen } 1776cac5fa39bd861861018bb2c3f0b36cc71c2caa38Jakob Stoklund Olesen DEBUG(dbgs() << '\n'); 1777a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen } 1778a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen 1779b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Since we allow local split results to be split again, there is a risk of 1780b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // creating infinite loops. It is tempting to require that the new live 1781b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // ranges have less instructions than the original. That would guarantee 1782b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // convergence, but it is too strict. A live range with 3 instructions can be 1783b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // split 2+3 (including the COPY), and we want to allow that. 1784b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // 1785b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Instead we use these rules: 1786b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // 178749743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // 1. Allow any split for ranges with getStage() < RS_Split2. (Except for the 1788b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // noop split, of course). 178949743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // 2. Require progress be made for ranges with getStage() == RS_Split2. All 1790b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // the new ranges must have fewer instructions than before the split. 179149743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // 3. New ranges with the same number of instructions are marked RS_Split2, 1792b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // smaller ranges are marked RS_New. 1793b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // 1794b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // These rules allow a 3 -> 2+3 split once, which we need. They also prevent 1795b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // excessive splitting and infinite loops. 1796b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // 179749743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen bool ProgressRequired = getStage(VirtReg) >= RS_Split2; 1798034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1799b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Best split candidate. 1800034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen unsigned BestBefore = NumGaps; 1801034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen unsigned BestAfter = 0; 1802034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen float BestDiff = 0; 1803034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 180403ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen const float blockFreq = 180503ef600623704e6dbf6ef0322062395e7b70bcd2Jakob Stoklund Olesen SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() * 180636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (1.0f / MBFI->getEntryFreq()); 1807034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen SmallVector<float, 8> GapWeight; 1808034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1809034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen Order.rewind(); 1810034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen while (unsigned PhysReg = Order.next()) { 1811034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Keep track of the largest spill weight that would need to be evicted in 1812034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // order to make use of PhysReg between UseSlots[i] and UseSlots[i+1]. 1813034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen calcGapWeights(PhysReg, GapWeight); 1814034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1815a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen // Remove any gaps with regmask clobbers. 1816042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen if (Matrix->checkRegMaskInterference(VirtReg, PhysReg)) 1817a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen for (unsigned i = 0, e = RegMaskGaps.size(); i != e; ++i) 1818eb3602472026dc029beb45ccbe09bc84162ba949Aaron Ballman GapWeight[RegMaskGaps[i]] = llvm::huge_valf; 1819a6d513f47467b90344a74624d4fb77673942e3ceJakob Stoklund Olesen 1820034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Try to find the best sequence of gaps to close. 1821034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // The new spill weight must be larger than any gap interference. 1822034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1823034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // We will split before Uses[SplitBefore] and after Uses[SplitAfter]. 1824b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen unsigned SplitBefore = 0, SplitAfter = 1; 1825034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1826034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // MaxGap should always be max(GapWeight[SplitBefore..SplitAfter-1]). 1827034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // It is the spill weight that needs to be evicted. 1828034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen float MaxGap = GapWeight[0]; 1829034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1830034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen for (;;) { 1831034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Live before/after split? 1832034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen const bool LiveBefore = SplitBefore != 0 || BI.LiveIn; 1833034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen const bool LiveAfter = SplitAfter != NumGaps || BI.LiveOut; 1834034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1835034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << PrintReg(PhysReg, TRI) << ' ' 1836034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen << Uses[SplitBefore] << '-' << Uses[SplitAfter] 1837034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen << " i=" << MaxGap); 1838034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1839034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Stop before the interval gets so big we wouldn't be making progress. 1840034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (!LiveBefore && !LiveAfter) { 1841034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << " all\n"); 1842034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1843034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1844034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Should the interval be extended or shrunk? 1845034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen bool Shrink = true; 1846034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1847b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // How many gaps would the new range have? 1848b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen unsigned NewGaps = LiveBefore + SplitAfter - SplitBefore + LiveAfter; 1849b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen 1850b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Legally, without causing looping? 1851b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen bool Legal = !ProgressRequired || NewGaps < NumGaps; 1852b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen 1853eb3602472026dc029beb45ccbe09bc84162ba949Aaron Ballman if (Legal && MaxGap < llvm::huge_valf) { 1854b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Estimate the new spill weight. Each instruction reads or writes the 1855b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // register. Conservatively assume there are no read-modify-write 1856b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // instructions. 1857034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // 1858b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // Try to guess the size of the new interval. 185937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const float EstWeight = normalizeSpillWeight( 186037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines blockFreq * (NewGaps + 1), 186137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Uses[SplitBefore].distance(Uses[SplitAfter]) + 186237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (LiveBefore + LiveAfter) * SlotIndex::InstrDist, 186337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1); 1864034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Would this split be possible to allocate? 1865034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Never allocate all gaps, we wouldn't be making progress. 186666446c803a11a26e4bb39b74091d146ac850ae4cJakob Stoklund Olesen DEBUG(dbgs() << " w=" << EstWeight); 186766446c803a11a26e4bb39b74091d146ac850ae4cJakob Stoklund Olesen if (EstWeight * Hysteresis >= MaxGap) { 1868034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen Shrink = false; 186966446c803a11a26e4bb39b74091d146ac850ae4cJakob Stoklund Olesen float Diff = EstWeight - MaxGap; 1870034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Diff > BestDiff) { 1871034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << " (best)"); 187266446c803a11a26e4bb39b74091d146ac850ae4cJakob Stoklund Olesen BestDiff = Hysteresis * Diff; 1873034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen BestBefore = SplitBefore; 1874034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen BestAfter = SplitAfter; 1875034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1876034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1877034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1878034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1879034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Try to shrink. 1880034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (Shrink) { 1881b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen if (++SplitBefore < SplitAfter) { 1882034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << " shrink\n"); 1883034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Recompute the max when necessary. 1884034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (GapWeight[SplitBefore - 1] >= MaxGap) { 1885034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen MaxGap = GapWeight[SplitBefore]; 1886034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen for (unsigned i = SplitBefore + 1; i != SplitAfter; ++i) 1887034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen MaxGap = std::max(MaxGap, GapWeight[i]); 1888034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1889034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen continue; 1890034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1891034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen MaxGap = 0; 1892034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1893034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1894034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Try to extend the interval. 1895034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (SplitAfter >= NumGaps) { 1896034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << " end\n"); 1897034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen break; 1898034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1899034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1900034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << " extend\n"); 1901b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen MaxGap = std::max(MaxGap, GapWeight[SplitAfter++]); 1902034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1903034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen } 1904034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1905034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Didn't find any candidates? 1906034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen if (BestBefore == NumGaps) 1907034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen return 0; 1908034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1909034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen DEBUG(dbgs() << "Best local split range: " << Uses[BestBefore] 1910034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen << '-' << Uses[BestAfter] << ", " << BestDiff 1911034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen << ", " << (BestAfter - BestBefore + 1) << " instrs\n"); 1912034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1913de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); 1914bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen SE->reset(LREdit); 1915bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen 1916bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen SE->openIntv(); 1917bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen SlotIndex SegStart = SE->enterIntvBefore(Uses[BestBefore]); 1918bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen SlotIndex SegStop = SE->leaveIntvAfter(Uses[BestAfter]); 1919bece06f0c6936527e2b1c72d09f7d3a949af9a47Jakob Stoklund Olesen SE->useIntv(SegStart, SegStop); 1920b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen SmallVector<unsigned, 8> IntvMap; 1921b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen SE->finish(&IntvMap); 19221feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey DebugVars->splitRegister(VirtReg.reg, LREdit.regs(), *LIS); 1923b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen 1924b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // If the new range has the same number of instructions as before, mark it as 192549743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // RS_Split2 so the next split will be forced to make progress. Otherwise, 1926b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen // leave the new intervals as RS_New so they can compete. 1927b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen bool LiveBefore = BestBefore != 0 || BI.LiveIn; 1928b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen bool LiveAfter = BestAfter != NumGaps || BI.LiveOut; 1929b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen unsigned NewGaps = LiveBefore + BestAfter - BestBefore + LiveAfter; 1930b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen if (NewGaps >= NumGaps) { 1931b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen DEBUG(dbgs() << "Tagging non-progress ranges: "); 1932b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen assert(!ProgressRequired && "Didn't make progress when it was required."); 1933b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen for (unsigned i = 0, e = IntvMap.size(); i != e; ++i) 1934b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen if (IntvMap[i] == 1) { 19351feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey setStage(LIS->getInterval(LREdit.get(i)), RS_Split2); 19361feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey DEBUG(dbgs() << PrintReg(LREdit.get(i))); 1937b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen } 1938b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen DEBUG(dbgs() << '\n'); 1939b3e705f88987608d4eb49668dac0e235d04df884Jakob Stoklund Olesen } 19400db841f9c2b9a25fb5ecb36e350d3a802c35654cJakob Stoklund Olesen ++NumLocalSplits; 1941034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1942034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen return 0; 1943034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen} 1944034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen 1945034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen//===----------------------------------------------------------------------===// 1946ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen// Live Range Splitting 1947ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen//===----------------------------------------------------------------------===// 1948ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1949ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen/// trySplit - Try to split VirtReg or one of its interferences, making it 1950ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen/// assignable. 1951ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen/// @return Physreg when VirtReg may be assigned and/or new NewVRegs. 1952ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesenunsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, 19531feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned>&NewVRegs) { 1954ccfa446450c9e3e0b3591343c4c5bea1e4cdc043Jakob Stoklund Olesen // Ranges must be Split2 or less. 1955ccfa446450c9e3e0b3591343c4c5bea1e4cdc043Jakob Stoklund Olesen if (getStage(VirtReg) >= RS_Spill) 1956ccfa446450c9e3e0b3591343c4c5bea1e4cdc043Jakob Stoklund Olesen return 0; 1957ccfa446450c9e3e0b3591343c4c5bea1e4cdc043Jakob Stoklund Olesen 1958034a80d065358b412cdd270e08fb6f1986e65e50Jakob Stoklund Olesen // Local intervals are handled separately. 1959a2ebf60ef2c434428af7f810b13327ab50245a67Jakob Stoklund Olesen if (LIS->intervalIsInOneMBB(VirtReg)) { 1960a2ebf60ef2c434428af7f810b13327ab50245a67Jakob Stoklund Olesen NamedRegionTimer T("Local Splitting", TimerGroupName, TimePassesIsEnabled); 196122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen SA->analyze(&VirtReg); 1962d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen unsigned PhysReg = tryLocalSplit(VirtReg, Order, NewVRegs); 1963d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen if (PhysReg || !NewVRegs.empty()) 1964d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return PhysReg; 1965d74d2847573df690b6a91254688ef3fd974f83f7Jakob Stoklund Olesen return tryInstructionSplit(VirtReg, Order, NewVRegs); 1966a2ebf60ef2c434428af7f810b13327ab50245a67Jakob Stoklund Olesen } 1967a2ebf60ef2c434428af7f810b13327ab50245a67Jakob Stoklund Olesen 1968a2ebf60ef2c434428af7f810b13327ab50245a67Jakob Stoklund Olesen NamedRegionTimer T("Global Splitting", TimerGroupName, TimePassesIsEnabled); 1969ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 197022a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen SA->analyze(&VirtReg); 197122a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 19727d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen // FIXME: SplitAnalysis may repair broken live ranges coming from the 19737d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen // coalescer. That may cause the range to become allocatable which means that 19747d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen // tryRegionSplit won't be making progress. This check should be replaced with 19757d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen // an assertion when the coalescer is fixed. 19767d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen if (SA->didRepairRange()) { 19777d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen // VirtReg has changed, so all cached queries are invalid. 1978042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen Matrix->invalidateVirtRegs(); 19797d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) 19807d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen return PhysReg; 19817d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen } 19827d6b6a05b549da70b4473f015c97954c2a422724Jakob Stoklund Olesen 198349743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // First try to split around a region spanning multiple blocks. RS_Split2 198449743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // ranges already made dubious progress with region splitting, so they go 198549743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen // straight to single block splitting. 198649743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen if (getStage(VirtReg) < RS_Split2) { 198749743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs); 198849743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen if (PhysReg || !NewVRegs.empty()) 198949743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen return PhysReg; 199049743b18f50ac0f7e065f4754a26965d4db388deJakob Stoklund Olesen } 1991ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 1992dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen // Then isolate blocks. 1993dab35d33ae17353cb01aaaa42abbcb28b33eb98aJakob Stoklund Olesen return tryBlockSplit(VirtReg, Order, NewVRegs); 1994ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen} 1995ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 199636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 199736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Last Chance Recoloring 199836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 199936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 200036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// mayRecolorAllInterferences - Check if the virtual registers that 200136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// interfere with \p VirtReg on \p PhysReg (or one of its aliases) may be 200236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recolored to free \p PhysReg. 200336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// When true is returned, \p RecoloringCandidates has been augmented with all 200436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the live intervals that need to be recolored in order to free \p PhysReg 200536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// for \p VirtReg. 200636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p FixedRegisters contains all the virtual registers that cannot be 200736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recolored. 200836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool 200936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesRAGreedy::mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg, 201036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallLISet &RecoloringCandidates, 201136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SmallVirtRegSet &FixedRegisters) { 201236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg); 201336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 201436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { 201536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); 201636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If there is LastChanceRecoloringMaxInterference or more interferences, 201736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // chances are one would not be recolorable. 201836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Q.collectInterferingVRegs(LastChanceRecoloringMaxInterference) >= 2019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LastChanceRecoloringMaxInterference && !ExhaustiveSearch) { 202036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Early abort: too many interferences.\n"); 2021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CutOffInfo |= CO_Interf; 202236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 202336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 202436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = Q.interferingVRegs().size(); i; --i) { 202536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveInterval *Intf = Q.interferingVRegs()[i - 1]; 202636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Intf is done and sit on the same register class as VirtReg, 202736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // it would not be recolorable as it is in the same state as VirtReg. 202836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((getStage(*Intf) == RS_Done && 202936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MRI->getRegClass(Intf->reg) == CurRC) || 203036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FixedRegisters.count(Intf->reg)) { 203136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Early abort: the inteference is not recolorable.\n"); 203236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 203336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 203436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RecoloringCandidates.insert(Intf); 203536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 203636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 203736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 203836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 203936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 204036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// tryLastChanceRecoloring - Try to assign a color to \p VirtReg by recoloring 204136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// its interferences. 204236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Last chance recoloring chooses a color for \p VirtReg and recolors every 204336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// virtual register that was using it. The recoloring process may recursively 204436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// use the last chance recoloring. Therefore, when a virtual register has been 204536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// assigned a color by this mechanism, it is marked as Fixed, i.e., it cannot 204636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// be last-chance-recolored again during this recoloring "session". 204736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// E.g., 204836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Let 204936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vA can use {R1, R2 } 205036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vB can use { R2, R3} 205136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vC can use {R1 } 205236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Where vA, vB, and vC cannot be split anymore (they are reloads for 205336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instance) and they all interfere. 205436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 205536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vA is assigned R1 205636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vB is assigned R2 205736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vC tries to evict vA but vA is already done. 205836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Regular register allocation fails. 205936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 206036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Last chance recoloring kicks in: 206136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vC does as if vA was evicted => vC uses R1. 206236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vC is marked as fixed. 206336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vA needs to find a color. 206436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// None are available. 206536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vA cannot evict vC: vC is a fixed virtual register now. 206636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vA does as if vB was evicted => vA uses R2. 206736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// vB needs to find a color. 206836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// R3 is available. 206936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Recoloring => vC = R1, vA = R2, vB = R3 207036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 207136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p Order defines the preferred allocation order for \p VirtReg. 207236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p NewRegs will contain any new virtual register that have been created 207336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (split, spill) during the process and that must be assigned. 207436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p FixedRegisters contains all the virtual registers that cannot be 207536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recolored. 207636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p Depth gives the current depth of the last chance recoloring. 207736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \return a physical register that can be used for VirtReg or ~0u if none 207836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// exists. 207936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesunsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, 208036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AllocationOrder &Order, 208136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs, 208236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &FixedRegisters, 208336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) { 208436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Try last chance recoloring for " << VirtReg << '\n'); 208536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Ranges must be Done. 208636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert((getStage(VirtReg) >= RS_Done || !VirtReg.isSpillable()) && 208736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "Last chance recoloring should really be last chance"); 208836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Set the max depth to LastChanceRecoloringMaxDepth. 208936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We may want to reconsider that if we end up with a too large search space 209036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for target with hundreds of registers. 209136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Indeed, in that case we may want to cut the search space earlier. 2092dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Depth >= LastChanceRecoloringMaxDepth && !ExhaustiveSearch) { 209336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Abort because max depth has been reached.\n"); 2094dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CutOffInfo |= CO_Depth; 209536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return ~0u; 209636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 209736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 209836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Set of Live intervals that will need to be recolored. 209936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallLISet RecoloringCandidates; 210036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Record the original mapping virtual register to physical register in case 210136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the recoloring fails. 210236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<unsigned, unsigned> VirtRegToPhysReg; 210336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Mark VirtReg as fixed, i.e., it will not be recolored pass this point in 210436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this recoloring "session". 210536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FixedRegisters.insert(VirtReg.reg); 210636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 210736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Order.rewind(); 210836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (unsigned PhysReg = Order.next()) { 210936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Try to assign: " << VirtReg << " to " 211036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << PrintReg(PhysReg, TRI) << '\n'); 211136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RecoloringCandidates.clear(); 211236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VirtRegToPhysReg.clear(); 211336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 211436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is only possible to recolor virtual register interference. 211536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Matrix->checkInterference(VirtReg, PhysReg) > 211636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveRegMatrix::IK_VirtReg) { 211736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Some inteferences are not with virtual registers.\n"); 211836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 211936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 212036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 212136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 212236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Early give up on this PhysReg if it is obvious we cannot recolor all 212336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the interferences. 212436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!mayRecolorAllInterferences(PhysReg, VirtReg, RecoloringCandidates, 212536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FixedRegisters)) { 212636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Some inteferences cannot be recolored.\n"); 212736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 212836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 212936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 213036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // RecoloringCandidates contains all the virtual registers that interfer 213136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // with VirtReg on PhysReg (or one of its aliases). 213236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Enqueue them for recoloring and perform the actual recoloring. 213336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PQueue RecoloringQueue; 213436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallLISet::iterator It = RecoloringCandidates.begin(), 213536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EndIt = RecoloringCandidates.end(); 213636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines It != EndIt; ++It) { 213736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ItVirtReg = (*It)->reg; 213836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines enqueue(RecoloringQueue, *It); 213936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(VRM->hasPhys(ItVirtReg) && 214036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "Interferences are supposed to be with allocated vairables"); 214136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Record the current allocation. 214336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VirtRegToPhysReg[ItVirtReg] = VRM->getPhys(ItVirtReg); 214436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // unset the related struct. 214536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->unassign(**It); 214636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 214736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do as if VirtReg was assigned to PhysReg so that the underlying 214936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // recoloring has the right information about the interferes and 215036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // available colors. 215136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->assign(VirtReg, PhysReg); 215236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 215336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Save the current recoloring state. 215436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we cannot recolor all the interferences, we will have to start again 215536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // at this point for the next physical register. 215636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet SaveFixedRegisters(FixedRegisters); 215736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (tryRecoloringCandidates(RecoloringQueue, NewVRegs, FixedRegisters, 215836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Depth)) { 215936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do not mess up with the global assignment process. 216036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // I.e., VirtReg must be unassigned. 216136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->unassign(VirtReg); 216236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return PhysReg; 216336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 216436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 216536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Fail to assign: " << VirtReg << " to " 216636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << PrintReg(PhysReg, TRI) << '\n'); 216736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 216836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The recoloring attempt failed, undo the changes. 216936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FixedRegisters = SaveFixedRegisters; 217036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->unassign(VirtReg); 217136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 217236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallLISet::iterator It = RecoloringCandidates.begin(), 217336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EndIt = RecoloringCandidates.end(); 217436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines It != EndIt; ++It) { 217536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ItVirtReg = (*It)->reg; 217636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VRM->hasPhys(ItVirtReg)) 217736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->unassign(**It); 2178f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned ItPhysReg = VirtRegToPhysReg[ItVirtReg]; 2179f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Matrix->assign(**It, ItPhysReg); 218036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 218136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 218236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 218336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Last chance recoloring did not worked either, give up. 218436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return ~0u; 218536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 218636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 218736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// tryRecoloringCandidates - Try to assign a new color to every register 218836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// in \RecoloringQueue. 218936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p NewRegs will contain any new virtual register created during the 219036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recoloring process. 219136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p FixedRegisters[in/out] contains all the registers that have been 219236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recolored. 219336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \return true if all virtual registers in RecoloringQueue were successfully 219436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// recolored, false otherwise. 219536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue, 219636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs, 219736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &FixedRegisters, 219836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) { 219936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!RecoloringQueue.empty()) { 220036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LiveInterval *LI = dequeue(RecoloringQueue); 220136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Try to recolor: " << *LI << '\n'); 220236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned PhysReg; 220336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PhysReg = selectOrSplitImpl(*LI, NewVRegs, FixedRegisters, Depth + 1); 220436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PhysReg == ~0u || !PhysReg) 220536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 220636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Recoloring of " << *LI 220736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << " succeeded with: " << PrintReg(PhysReg, TRI) << '\n'); 220836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matrix->assign(*LI, PhysReg); 220936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FixedRegisters.insert(LI->reg); 221036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 221136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 221236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 2213ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 2214b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen//===----------------------------------------------------------------------===// 2215770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen// Main Entry Point 2216770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen//===----------------------------------------------------------------------===// 2217770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen 2218770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesenunsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg, 22191feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey SmallVectorImpl<unsigned> &NewVRegs) { 2220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CutOffInfo = CO_None; 2221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LLVMContext &Ctx = MF->getFunction()->getContext(); 222236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet FixedRegisters; 2223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters); 2224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Reg == ~0U && (CutOffInfo != CO_None)) { 2225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint8_t CutOffEncountered = CutOffInfo & (CO_Depth | CO_Interf); 2226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CutOffEncountered == CO_Depth) 2227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ctx.emitError("register allocation failed: maximum depth for recoloring " 2228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "reached. Use -fexhaustive-register-search to skip " 2229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "cutoffs"); 2230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (CutOffEncountered == CO_Interf) 2231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ctx.emitError("register allocation failed: maximum interference for " 2232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "recoloring reached. Use -fexhaustive-register-search " 2233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "to skip cutoffs"); 2234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (CutOffEncountered == (CO_Depth | CO_Interf)) 2235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ctx.emitError("register allocation failed: maximum interference and " 2236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "depth for recoloring reached. Use " 2237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "-fexhaustive-register-search to skip cutoffs"); 2238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Reg; 224036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 224136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 224236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Using a CSR for the first time has a cost because it causes push|pop 224336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// to be added to prologue|epilogue. Splitting a cold section of the live 224436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// range can have lower cost than using the CSR for the first time; 224536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Spilling a live range in the cold path can have lower cost than using 224636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the CSR for the first time. Returns the physical register if we decide 224736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// to use the CSR; otherwise return 0. 224836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesunsigned RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg, 224936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AllocationOrder &Order, 225036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned PhysReg, 225136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &CostPerUseLimit, 225236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs) { 225336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) { 225436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We choose spill over using the CSR for the first time if the spill cost 225536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // is lower than CSRCost. 225636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SA->analyze(&VirtReg); 225736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (calcSpillCost() >= CSRCost) 225836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return PhysReg; 225936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 226036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We are going to spill, set CostPerUseLimit to 1 to make sure that 226136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we will not use a callee-saved register in tryEvict. 226236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CostPerUseLimit = 1; 226336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 0; 226436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 226536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (getStage(VirtReg) < RS_Split) { 226636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We choose pre-splitting over using the CSR for the first time if 226736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the cost of splitting is lower than CSRCost. 226836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SA->analyze(&VirtReg); 226936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumCands = 0; 2270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BlockFrequency BestCost = CSRCost; // Don't modify CSRCost. 2271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, BestCost, 2272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumCands, true /*IgnoreCSR*/); 227336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BestCand == NoCand) 227436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use the CSR if we can't find a region split below CSRCost. 227536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return PhysReg; 227636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 227736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Perform the actual pre-splitting. 227836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines doRegionSplit(VirtReg, BestCand, false/*HasCompact*/, NewVRegs); 227936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 0; 228036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 228136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return PhysReg; 228236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 228336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2284ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesvoid RAGreedy::aboutToRemoveInterval(LiveInterval &LI) { 2285ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Do not keep invalid information around. 2286ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SetOfBrokenHints.remove(&LI); 2287ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2288ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid RAGreedy::initializeCSRCost() { 2290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We use the larger one out of the command-line option and the value report 2291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // by TRI. 2292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CSRCost = BlockFrequency( 2293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::max((unsigned)CSRFirstTimeCost, TRI->getCSRFirstUseCost())); 2294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CSRCost.getFrequency()) 2295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 2296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Raw cost is relative to Entry == 2^14; scale it appropriately. 2298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t ActualEntry = MBFI->getEntryFreq(); 2299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ActualEntry) { 2300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CSRCost = 0; 2301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 2302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t FixedEntry = 1 << 14; 2304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ActualEntry < FixedEntry) 2305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CSRCost *= BranchProbability(ActualEntry, FixedEntry); 2306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (ActualEntry <= UINT32_MAX) 2307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Invert the fraction and divide. 2308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CSRCost /= BranchProbability(FixedEntry, ActualEntry); 2309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 2310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Can't use BranchProbability in general, since it takes 32-bit numbers. 2311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CSRCost = CSRCost.getFrequency() * (ActualEntry / FixedEntry); 2312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 2313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2314ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Collect the hint info for \p Reg. 2315ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// The results are stored into \p Out. 2316ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \p Out is not cleared before being populated. 2317ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesvoid RAGreedy::collectHintInfo(unsigned Reg, HintsInfo &Out) { 2318ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (const MachineInstr &Instr : MRI->reg_nodbg_instructions(Reg)) { 2319ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!Instr.isFullCopy()) 2320ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2321ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Look for the other end of the copy. 2322ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned OtherReg = Instr.getOperand(0).getReg(); 2323ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (OtherReg == Reg) { 2324ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OtherReg = Instr.getOperand(1).getReg(); 2325ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (OtherReg == Reg) 2326ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2327ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2328ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Get the current assignment. 2329ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned OtherPhysReg = TargetRegisterInfo::isPhysicalRegister(OtherReg) 2330ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ? OtherReg 2331ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : VRM->getPhys(OtherReg); 2332ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Push the collected information. 2333ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Out.push_back(HintInfo(MBFI->getBlockFreq(Instr.getParent()), OtherReg, 2334ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OtherPhysReg)); 2335ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2336ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2337ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2338ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Using the given \p List, compute the cost of the broken hints if 2339ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \p PhysReg was used. 2340ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \return The cost of \p List for \p PhysReg. 2341ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesBlockFrequency RAGreedy::getBrokenHintFreq(const HintsInfo &List, 2342ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned PhysReg) { 2343ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BlockFrequency Cost = 0; 2344ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (const HintInfo &Info : List) { 2345ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Info.PhysReg != PhysReg) 2346ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Cost += Info.Freq; 2347ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2348ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return Cost; 2349ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2350ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2351ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Using the register assigned to \p VirtReg, try to recolor 2352ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// all the live ranges that are copy-related with \p VirtReg. 2353ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// The recoloring is then propagated to all the live-ranges that have 2354ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// been recolored and so on, until no more copies can be coalesced or 2355ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// it is not profitable. 2356ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// For a given live range, profitability is determined by the sum of the 2357ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// frequencies of the non-identity copies it would introduce with the old 2358ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// and new register. 2359ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesvoid RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) { 2360ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We have a broken hint, check if it is possible to fix it by 2361ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // reusing PhysReg for the copy-related live-ranges. Indeed, we evicted 2362ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // some register and PhysReg may be available for the other live-ranges. 2363ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallSet<unsigned, 4> Visited; 2364ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<unsigned, 2> RecoloringCandidates; 2365ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines HintsInfo Info; 2366ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned Reg = VirtReg.reg; 2367ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned PhysReg = VRM->getPhys(Reg); 2368ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Start the recoloring algorithm from the input live-interval, then 2369ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // it will propagate to the ones that are copy-related with it. 2370ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Visited.insert(Reg); 2371ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines RecoloringCandidates.push_back(Reg); 2372ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2373ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "Trying to reconcile hints for: " << PrintReg(Reg, TRI) << '(' 2374ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines << PrintReg(PhysReg, TRI) << ")\n"); 2375ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2376ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines do { 2377ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Reg = RecoloringCandidates.pop_back_val(); 2378ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2379ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We cannot recolor physcal register. 2380ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TargetRegisterInfo::isPhysicalRegister(Reg)) 2381ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2382ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2383ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(VRM->hasPhys(Reg) && "We have unallocated variable!!"); 2384ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2385ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Get the live interval mapped with this virtual register to be able 2386ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // to check for the interference with the new color. 2387ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LiveInterval &LI = LIS->getInterval(Reg); 2388ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned CurrPhys = VRM->getPhys(Reg); 2389ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check that the new color matches the register class constraints and 2390ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // that it is free for this live range. 2391ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (CurrPhys != PhysReg && (!MRI->getRegClass(Reg)->contains(PhysReg) || 2392ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Matrix->checkInterference(LI, PhysReg))) 2393ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2394ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2395ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << PrintReg(Reg, TRI) << '(' << PrintReg(CurrPhys, TRI) 2396ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines << ") is recolorable.\n"); 2397ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2398ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Gather the hint info. 2399ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Info.clear(); 2400ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines collectHintInfo(Reg, Info); 2401ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check if recoloring the live-range will increase the cost of the 2402ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // non-identity copies. 2403ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (CurrPhys != PhysReg) { 2404ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "Checking profitability:\n"); 2405ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BlockFrequency OldCopiesCost = getBrokenHintFreq(Info, CurrPhys); 2406ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BlockFrequency NewCopiesCost = getBrokenHintFreq(Info, PhysReg); 2407ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "Old Cost: " << OldCopiesCost.getFrequency() 2408ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines << "\nNew Cost: " << NewCopiesCost.getFrequency() << '\n'); 2409ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (OldCopiesCost < NewCopiesCost) { 2410ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "=> Not profitable.\n"); 2411ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2412ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2413ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // At this point, the cost is either cheaper or equal. If it is 2414ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // equal, we consider this is profitable because it may expose 2415ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // more recoloring opportunities. 2416ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "=> Profitable.\n"); 2417ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Recolor the live-range. 2418ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Matrix->unassign(LI); 2419ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Matrix->assign(LI, PhysReg); 2420ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2421ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Push all copy-related live-ranges to keep reconciling the broken 2422ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // hints. 2423ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (const HintInfo &HI : Info) { 2424ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Visited.insert(HI.Reg).second) 2425ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines RecoloringCandidates.push_back(HI.Reg); 2426ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2427ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } while (!RecoloringCandidates.empty()); 2428ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2429ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2430ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Try to recolor broken hints. 2431ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Broken hints may be repaired by recoloring when an evicted variable 2432ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// freed up a register for a larger live-range. 2433ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Consider the following example: 2434ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB1: 2435ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// a = 2436ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// b = 2437ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB2: 2438ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// ... 2439ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = b 2440ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = a 2441ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Let us assume b gets split: 2442ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB1: 2443ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// a = 2444ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// b = 2445ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB2: 2446ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// c = b 2447ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// ... 2448ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// d = c 2449ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = d 2450ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = a 2451ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Because of how the allocation work, b, c, and d may be assigned different 2452ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// colors. Now, if a gets evicted later: 2453ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB1: 2454ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// a = 2455ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// st a, SpillSlot 2456ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// b = 2457ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// BB2: 2458ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// c = b 2459ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// ... 2460ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// d = c 2461ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = d 2462ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// e = ld SpillSlot 2463ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// = e 2464ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// This is likely that we can assign the same register for b, c, and d, 2465ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// getting rid of 2 copies. 2466ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesvoid RAGreedy::tryHintsRecoloring() { 2467ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (LiveInterval *LI : SetOfBrokenHints) { 2468ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(TargetRegisterInfo::isVirtualRegister(LI->reg) && 2469ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "Recoloring is possible only for virtual registers"); 2470ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Some dead defs may be around (e.g., because of debug uses). 2471ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Ignore those. 2472ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!VRM->hasPhys(LI->reg)) 2473ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2474ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines tryHintRecoloring(*LI); 2475ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2476ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2477ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 247836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesunsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, 247936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<unsigned> &NewVRegs, 248036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVirtRegSet &FixedRegisters, 248136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) { 248236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CostPerUseLimit = ~0u; 2483770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen // First try assigning a free register. 2484f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix); 248536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) { 248636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // When NewVRegs is not empty, we may have made decisions such as evicting 248736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // a virtual register, go with the earlier decisions and use the physical 248836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // register. 2489f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) && 2490f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewVRegs.empty()) { 249136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg, 249236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CostPerUseLimit, NewVRegs); 249336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CSRReg || !NewVRegs.empty()) 249436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return now if we decide to use a CSR or create new vregs due to 249536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // pre-splitting. 249636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return CSRReg; 249736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else 249836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return PhysReg; 249936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 2500b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trick 2501b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen LiveRangeStage Stage = getStage(VirtReg); 25021a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen DEBUG(dbgs() << StageName[Stage] 25031a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen << " Cascade " << ExtraRegInfo[VirtReg.reg].Cascade << '\n'); 2504b8d936bc179ddf31b6350015d74900b74db6b450Jakob Stoklund Olesen 250576395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen // Try to evict a less worthy live range, but only for ranges from the primary 2506fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen // queue. The RS_Split ranges already failed to do this, and they should not 250776395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen // get a second chance until they have been split. 2508fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen if (Stage != RS_Split) 2509ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (unsigned PhysReg = 2510ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit)) { 2511ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned Hint = MRI->getSimpleHint(VirtReg.reg); 2512ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If VirtReg has a hint and that hint is broken record this 2513ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // virtual register as a recoloring candidate for broken hint. 2514ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Indeed, since we evicted a variable in its neighborhood it is 2515ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // likely we can at least partially recolor some of the 2516ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // copy-related live-ranges. 2517ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Hint && Hint != PhysReg) 2518ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SetOfBrokenHints.insert(&VirtReg); 251976395c9a31444fa86514473e0852cdd67752d0e8Jakob Stoklund Olesen return PhysReg; 2520ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2521b853e6c3702149cdbbd6fa404334e3ba0055641aAndrew Trick 2522ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen assert(NewVRegs.empty() && "Cannot append to existing NewVRegs"); 2523ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen 2524107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen // The first time we see a live range, don't try to split or spill. 2525107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen // Wait until the second time, when all smaller ranges have been allocated. 2526107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen // This gives a better picture of the interference to split around. 2527fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen if (Stage < RS_Split) { 2528fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen setStage(VirtReg, RS_Split); 2529c1655e1a3c3a566b91b0513b56d61b58da1e36baJakob Stoklund Olesen DEBUG(dbgs() << "wait for second round\n"); 25301feb5854aeeda897e9318c8193d187673c8576b8Mark Lacey NewVRegs.push_back(VirtReg.reg); 2531107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen return 0; 2532107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen } 2533107d366df762c18294dc00f5de916f62672353ffJakob Stoklund Olesen 2534bf4e10f2f69db24c107cb61d6fe10ed5b2047374Jakob Stoklund Olesen // If we couldn't allocate a register from spilling, there is probably some 2535bf4e10f2f69db24c107cb61d6fe10ed5b2047374Jakob Stoklund Olesen // invalid inline assembly. The base class wil report it. 2536fa89a0344bba7a0ae87d3de204d18bb1ecaa5955Jakob Stoklund Olesen if (Stage >= RS_Done || !VirtReg.isSpillable()) 253736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return tryLastChanceRecoloring(VirtReg, Order, NewVRegs, FixedRegisters, 253836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Depth); 253922a1df6bf24c188dd637a0bb2cf9a2648806b6b1Jakob Stoklund Olesen 254046c83c80c5737743c955ff007fa6409804a7abf0Jakob Stoklund Olesen // Try splitting VirtReg or interferences. 2541ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs); 2542ccdb3fcef9aeb9f683cd738afbe1cd961bb0c1efJakob Stoklund Olesen if (PhysReg || !NewVRegs.empty()) 2543b64d92e29f38002e52a22fe36ea2d488968e3537Jakob Stoklund Olesen return PhysReg; 2544b64d92e29f38002e52a22fe36ea2d488968e3537Jakob Stoklund Olesen 2545770d42de3b7643b2b4f835f32e3a16275b9fbdbaJakob Stoklund Olesen // Finally spill VirtReg itself. 2546f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EnableDeferredSpilling && getStage(VirtReg) < RS_Memory) { 2547f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: This is experimental and in particular, we do not model 2548f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the live range splitting done by spilling correctly. 2549f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We would need a deep integration with the spiller to do the 2550f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // right thing here. Anyway, that is still good for early testing. 2551f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setStage(VirtReg, RS_Memory); 2552f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << "Do as if this register is in memory\n"); 2553f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewVRegs.push_back(VirtReg.reg); 2554f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else { 2555f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled); 2556de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); 2557f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar spiller().spill(LRE); 2558f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done); 2559cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 2560f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (VerifyEnabled) 2561f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MF->verify(this, "After spilling"); 2562f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 2563c46570dc05851395829bef904bb6ddb1260400d1Jakob Stoklund Olesen 2564cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen // The live virtual register requesting allocation was spilled, so tell 2565cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen // the caller not to allocate anything during this round. 2566cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen return 0; 2567cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 2568cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 2569cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesenbool RAGreedy::runOnMachineFunction(MachineFunction &mf) { 2570cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n" 2571986d76d7b3844b9a2f3d01a48975952749267a93David Blaikie << "********** Function: " << mf.getName() << '\n'); 2572cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen 2573cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen MF = &mf; 257437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TRI = MF->getSubtarget().getRegisterInfo(); 257537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TII = MF->getSubtarget().getInstrInfo(); 257636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RCI.runOnMachineFunction(mf); 2577c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 2578c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EnableLocalReassign = EnableLocalReassignment || 257937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MF->getSubtarget().enableRALocalReassignment( 258037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MF->getTarget().getOptLevel()); 2581c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 2582af24964251e27c2dd863239ba66ffd967b593be5Jakob Stoklund Olesen if (VerifyEnabled) 258389cab93fe999f6d81b4b99a71ac797b7ecfec277Jakob Stoklund Olesen MF->verify(this, "Before greedy register allocator"); 2584af24964251e27c2dd863239ba66ffd967b593be5Jakob Stoklund Olesen 2585d4348a2dc24c4fb012c1b9b20e71908f52049283Jakob Stoklund Olesen RegAllocBase::init(getAnalysis<VirtRegMap>(), 2586d4348a2dc24c4fb012c1b9b20e71908f52049283Jakob Stoklund Olesen getAnalysis<LiveIntervals>(), 2587d4348a2dc24c4fb012c1b9b20e71908f52049283Jakob Stoklund Olesen getAnalysis<LiveRegMatrix>()); 2588b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen Indexes = &getAnalysis<SlotIndexes>(); 25894eed756153b84c211114a3e9186bf0cb55d4b394Benjamin Kramer MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); 2590f428eb6c1b09a2322b7a577b0bf2e49dd107bceaJakob Stoklund Olesen DomTree = &getAnalysis<MachineDominatorTree>(); 2591f6dff84d4e44d6c4a46c4f8a18e13c78f804547cJakob Stoklund Olesen SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM)); 2592d0bb5e2ca05d9c942223bf05e5940bb0c6cc9d3fJakob Stoklund Olesen Loops = &getAnalysis<MachineLoopInfo>(); 2593b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen Bundles = &getAnalysis<EdgeBundles>(); 2594b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen SpillPlacer = &getAnalysis<SpillPlacement>(); 2595f42b66169d75301346e3685fd2b3e45e47806367Jakob Stoklund Olesen DebugVars = &getAnalysis<LiveDebugVariables>(); 2596de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 2597b5fa9333431673aac2ced8dea80152349a85cf6fJakob Stoklund Olesen 2598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines initializeCSRCost(); 2599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2600f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar calculateSpillWeightsAndHints(*LIS, mf, VRM, *Loops, *MBFI); 2601a77da0579bc141eba62760e21a216e5d3eafd792Arnaud A. de Grandmaison 26025dca61397878a7207986455b42fa2117fbf78000Andrew Trick DEBUG(LIS->dump()); 26035dca61397878a7207986455b42fa2117fbf78000Andrew Trick 26041b847deb26b52051de39f4cbecd224c9fbd0d1c2Jakob Stoklund Olesen SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops)); 2605de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SE.reset(new SplitEditor(*SA, *AA, *LIS, *VRM, *DomTree, *MBFI)); 26061a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.clear(); 26071a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen ExtraRegInfo.resize(MRI->getNumVirtRegs()); 26081a988004dba412deb5d6b8e93b955dfc837065f0Jakob Stoklund Olesen NextCascade = 1; 2609042888db2bb195c86bf34afbb6907d70855d2830Jakob Stoklund Olesen IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI); 261000005782fa860f4b48b3b5261d92541c61ee2495Jakob Stoklund Olesen GlobalCand.resize(32); // This will grow as needed. 2611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SetOfBrokenHints.clear(); 2612d0bb5e2ca05d9c942223bf05e5940bb0c6cc9d3fJakob Stoklund Olesen 2613cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen allocatePhysRegs(); 2614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines tryHintsRecoloring(); 2615de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar postOptimization(); 2616de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2617cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen releaseMemory(); 2618cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen return true; 2619cba2e06d525b723849cd8e1f083eb1e59a494b4eJakob Stoklund Olesen} 2620