SelectionDAGISel.cpp revision d725f04ff81db0e61baa153bf21ca2dd08838244
1b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff//===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===// 277cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// 377cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// The LLVM Compiler Infrastructure 477cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 777cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// 877cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner//===----------------------------------------------------------------------===// 977cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// 1077cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// This implements the SelectionDAGISel class. 1177cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner// 1277cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner//===----------------------------------------------------------------------===// 1377cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner 149b414d3e2d0cb84512b55a3275a98490b090162aDaniel Dunbar#define DEBUG_TYPE "isel" 158a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner#include "ScheduleDAGSDNodes.h" 1677cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#include "SelectionDAGBuilder.h" 1777cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#include "FunctionLoweringInfo.h" 188599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff#include "llvm/CodeGen/SelectionDAGISel.h" 198a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner#include "llvm/Analysis/AliasAnalysis.h" 20ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff#include "llvm/Analysis/DebugInfo.h" 210750618b0beacdc9b0a9e210a661e4746823ced7Chris Lattner#include "llvm/Constants.h" 2226de4655a38b899e0f0dcef4175032175854d1cfChris Lattner#include "llvm/Function.h" 236cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer#include "llvm/InlineAsm.h" 246cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer#include "llvm/Instructions.h" 25158ecb9767faf87c2a33df3baec1b160dcc0be84Chris Lattner#include "llvm/Intrinsics.h" 2626e4cd331c389a9b7d51c91981a7a10c8a909bf4Fariborz Jahanian#include "llvm/IntrinsicInst.h" 27a95d3750441ac8ad03e36af8e6e74039c9a3109dTed Kremenek#include "llvm/LLVMContext.h" 28ab10b2ebad09c283ccab0ef043118b3cf0166b56Fariborz Jahanian#include "llvm/CodeGen/FastISel.h" 2972952fc11f80c975492a2a1e0f6e3601c5252e0aFariborz Jahanian#include "llvm/CodeGen/GCStrategy.h" 3077cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#include "llvm/CodeGen/GCMetadata.h" 31158ecb9767faf87c2a33df3baec1b160dcc0be84Chris Lattner#include "llvm/CodeGen/MachineFunction.h" 3277cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#include "llvm/CodeGen/MachineInstrBuilder.h" 3377cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#include "llvm/CodeGen/MachineModuleInfo.h" 34b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff#include "llvm/CodeGen/MachineRegisterInfo.h" 3573e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/CodeGen/ScheduleHazardRecognizer.h" 3673e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/CodeGen/SchedulerRegistry.h" 3773e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/CodeGen/SelectionDAG.h" 3873e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetRegisterInfo.h" 3973e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetIntrinsicInfo.h" 4073e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetInstrInfo.h" 4173e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetLowering.h" 4273e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetMachine.h" 4373e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Target/TargetOptions.h" 4473e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Support/Compiler.h" 4573e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Support/Debug.h" 4673e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Support/ErrorHandling.h" 4773e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Support/Timer.h" 4873e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/Support/raw_ostream.h" 4973e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include "llvm/ADT/Statistic.h" 5073e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian#include <algorithm> 5173e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanianusing namespace llvm; 5273e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian 5373e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz JahanianSTATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on"); 5473e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz JahanianSTATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path"); 5573e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanian 5673e437bbe7e9631ad9055b1d50f4ae8564efbdf3Fariborz Jahanianstatic cl::opt<bool> 572c64b7b9381be4ff62fbdc404ed3f14c8086898dChris LattnerEnableFastISelVerbose("fast-isel-verbose", cl::Hidden, 58e365c50a21cb02e0219433db0c4461566ad4a597Chris Lattner cl::desc("Enable verbose messages in the \"fast\" " 594f943c23a47d042c3275e78f2d6015daa650f105Steve Naroff "instruction selector")); 60f69cc5d6606fc65a76e3acd6eb6e13efd0098295Steve Naroffstatic cl::opt<bool> 618c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve NaroffEnableFastISelAbort("fast-isel-abort", cl::Hidden, 621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump cl::desc("Enable abort calls when \"fast\" instruction fails")); 6301c5748c29e75b29cab5fc7d8ad1b173b29c7ecfChris Lattner 6477cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner#ifndef NDEBUG 65ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidisstatic cl::opt<bool> 662b2453a7d8fe732561795431f39ceb2b2a832d84Chris LattnerViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, 6726de4655a38b899e0f0dcef4175032175854d1cfChris Lattner cl::desc("Pop up a window to show dags before the first " 682c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner "dag combine pass")); 691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic cl::opt<bool> 70a526c5c67e5a0473c340903ee542ce570119665fTed KremenekViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, 71a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek cl::desc("Pop up a window to show dags before legalize types")); 72a526c5c67e5a0473c340903ee542ce570119665fTed Kremenekstatic cl::opt<bool> 73fbfe82527dadb09c95609a742744d75a2322850cSteve NaroffViewLegalizeDAGs("view-legalize-dags", cl::Hidden, 74a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek cl::desc("Pop up a window to show dags before legalize")); 75a526c5c67e5a0473c340903ee542ce570119665fTed Kremenekstatic cl::opt<bool> 76e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz JahanianViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, 77e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian cl::desc("Pop up a window to show dags before the second " 78621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff "dag combine pass")); 79621edce9cd7d4e06979daf911cc306350619f33bSteve Naroffstatic cl::opt<bool> 80ab10b2ebad09c283ccab0ef043118b3cf0166b56Fariborz JahanianViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, 81ab10b2ebad09c283ccab0ef043118b3cf0166b56Fariborz Jahanian cl::desc("Pop up a window to show dags before the post legalize types" 82ab10b2ebad09c283ccab0ef043118b3cf0166b56Fariborz Jahanian " dag combine pass")); 83d82a9ab4540899e24c96a389c5488381c5551c78Steve Naroffstatic cl::opt<bool> 841eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpViewISelDAGs("view-isel-dags", cl::Hidden, 85ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff cl::desc("Pop up a window to show isel dags as they are selected")); 86874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroffstatic cl::opt<bool> 8780a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz JahanianViewSchedDAGs("view-sched-dags", cl::Hidden, 8880a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian cl::desc("Pop up a window to show sched dags as they are processed")); 89acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanianstatic cl::opt<bool> 90ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve NaroffViewSUnitDAGs("view-sunit-dags", cl::Hidden, 919bcb5fc1fd48c1f40c6a3b5a59130ebc313b4957Steve Naroff cl::desc("Pop up a window to show SUnit dags after they are processed")); 92d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian#else 93934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroffstatic const bool ViewDAGCombine1 = false, 949698464266660346fa2da1bddc3e6404a9819b25Steve Naroff ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false, 95c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff ViewDAGCombine2 = false, 961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump ViewDAGCombineLT = false, 97beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff ViewISelDAGs = false, ViewSchedDAGs = false, 98248a753f6b670692523c99afaeb8fe98f7ae3ca7Steve Naroff ViewSUnitDAGs = false; 99beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff#endif 1001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 101b586cce95c69161c771f7a966383beb4fc5dd394Fariborz Jahanian//===---------------------------------------------------------------------===// 102e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian/// 1031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// RegisterScheduler class - Track the registration of instruction schedulers. 104874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff/// 10554055232a5ddb9529726e934301b125cb720a273Steve Naroff//===---------------------------------------------------------------------===// 106874e232a0d5e179197de860e6dfa3e99cd42ad30Steve NaroffMachinePassRegistry RegisterScheduler::Registry; 107d82a9ab4540899e24c96a389c5488381c5551c78Steve Naroff 1081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump//===---------------------------------------------------------------------===// 109621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff/// 110621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff/// ISHeuristic command line option for instruction schedulers. 1111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// 112b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian//===---------------------------------------------------------------------===// 113b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanianstatic cl::opt<RegisterScheduler::FunctionPassCtor, false, 1141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump RegisterPassParser<RegisterScheduler> > 115a7b402dc258bf38ab5e206dbf4916a69d3ee3cc8Steve NaroffISHeuristic("pre-RA-sched", 11666d6f048471aad6daf67b3ea804b54fbea72a076Eli Friedman cl::init(&createDefaultScheduler), 117c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman cl::desc("Instruction schedulers available (before register" 118c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman " allocation):")); 119f292fcf6bd5118499a830e0950429effeb373c28Fariborz Jahanian 120c6d656e2b0491f224ddd48507627b51d630d749aEli Friedmanstatic RegisterScheduler 121ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve NaroffdefaultListDAGScheduler("default", "Best scheduler for the target", 12254055232a5ddb9529726e934301b125cb720a273Steve Naroff createDefaultScheduler); 12354055232a5ddb9529726e934301b125cb720a273Steve Naroff 12454055232a5ddb9529726e934301b125cb720a273Steve Naroffnamespace llvm { 1255e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian //===--------------------------------------------------------------------===// 1265e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian /// createDefaultScheduler - This creates an instruction scheduler appropriate 1275e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian /// for the target. 12854055232a5ddb9529726e934301b125cb720a273Steve Naroff ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS, 1291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump CodeGenOpt::Level OptLevel) { 13054055232a5ddb9529726e934301b125cb720a273Steve Naroff const TargetLowering &TLI = IS->getTargetLowering(); 131bab71685568085b635f077ee5720d22dffab84beFariborz Jahanian 132bab71685568085b635f077ee5720d22dffab84beFariborz Jahanian if (OptLevel == CodeGenOpt::None) 133bab71685568085b635f077ee5720d22dffab84beFariborz Jahanian return createFastDAGScheduler(IS, OptLevel); 134bab71685568085b635f077ee5720d22dffab84beFariborz Jahanian if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) 135a73165e47aefbea60312d284343660c9c962c9c3Fariborz Jahanian return createTDListDAGScheduler(IS, OptLevel); 13654055232a5ddb9529726e934301b125cb720a273Steve Naroff assert(TLI.getSchedulingPreference() == 1376cb6eb4c792b504ad652d9230640656852e18ee9Fariborz Jahanian TargetLowering::SchedulingForRegPressure && "Unknown sched type!"); 1386cb6eb4c792b504ad652d9230640656852e18ee9Fariborz Jahanian return createBURRListDAGScheduler(IS, OptLevel); 13954055232a5ddb9529726e934301b125cb720a273Steve Naroff } 14054055232a5ddb9529726e934301b125cb720a273Steve Naroff} 141c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff 142c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff// EmitInstrWithCustomInserter - This method should be implemented by targets 1438599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff// that mark instructions with the 'usesCustomInserter' flag. These 1448599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff// instructions are special in various ways, which require special support to 1458599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff// insert. The specified MachineInstr is created but not inserted into any 1461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump// basic blocks, and this method is called to expand it into a sequence of 1474c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff// instructions, potentially also creating new basic blocks and control flow. 1484c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff// When new basic blocks are inserted and the edges from MBB to its successors 1494c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff// are modified, the method should insert pairs of <OldSucc, NewSucc> into the 1504c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff// DenseMap. 15115f081de2c8ac7deadf5d938b458b20732230cd9Steve NaroffMachineBasicBlock * 15254055232a5ddb9529726e934301b125cb720a273Steve NaroffTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 153abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian MachineBasicBlock *MBB) const { 1548e2f57ad06d73a6c2cacf0167a3b9e244439a9f9Steve Naroff#ifndef NDEBUG 1551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump dbgs() << "If a target marks an instruction with " 156b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff "'usesCustomInserter', it must implement " 1571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump "TargetLowering::EmitInstrWithCustomInserter!"; 158545b9ae381158761d387d2b5b071f28b1f489ed6Fariborz Jahanian#endif 15977cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner llvm_unreachable(0); 160e3a6198400453c0d9623207718e4942f7c111f87Ted Kremenek return 0; 161e3a6198400453c0d9623207718e4942f7c111f87Ted Kremenek} 162f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 163682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattner//===----------------------------------------------------------------------===// 164682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattner// SelectionDAGISel code 165682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattner//===----------------------------------------------------------------------===// 166682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattner 167682bf92db408a6cbc3d37b5496a99b6ef85041ecChris LattnerSelectionDAGISel::SelectionDAGISel(const TargetMachine &tm, CodeGenOpt::Level OL) : 1682c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner MachineFunctionPass(&ID), TM(tm), TLI(*tm.getTargetLowering()), 16966d6f048471aad6daf67b3ea804b54fbea72a076Eli Friedman FuncInfo(new FunctionLoweringInfo(TLI)), 170c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman CurDAG(new SelectionDAG(tm, *FuncInfo)), 171c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), 172e452e0ffc81c6c3d79680f552f2623e6cf0956d7Ted Kremenek GFI(), 173e452e0ffc81c6c3d79680f552f2623e6cf0956d7Ted Kremenek OptLevel(OL), 1741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump DAGSize(0) 175dacbc5d46aaeea817742b8c76c987b87d2490c85Chris Lattner{} 1761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17788906cddbb1d5b3a868eeeec6cb170befc829c2fFariborz JahanianSelectionDAGISel::~SelectionDAGISel() { 1784c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff delete SDB; 1791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump delete CurDAG; 1804c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff delete FuncInfo; 1814c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff} 1824c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff 183b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroffvoid SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { 184b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff AU.addRequired<AliasAnalysis>(); 185b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff AU.addPreserved<AliasAnalysis>(); 186f3dd57e5378bcf5fc9f05832e92479a535d9cd8aChris Lattner AU.addRequired<GCModuleInfo>(); 18788906cddbb1d5b3a868eeeec6cb170befc829c2fFariborz Jahanian AU.addPreserved<GCModuleInfo>(); 1884c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff MachineFunctionPass::getAnalysisUsage(AU); 1894c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff} 1904c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff 1914c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroffbool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { 192dcbc5b0b0722282a0fdd829359fe0d7e22adb882Chris Lattner // Do some sanity-checking on the command-line options. 1930a14eee528a901c16f0e288fbc10a3abc1660d87Chris Lattner assert((!EnableFastISelVerbose || EnableFastISel) && 1940a14eee528a901c16f0e288fbc10a3abc1660d87Chris Lattner "-fast-isel-verbose requires -fast-isel"); 195dcbc5b0b0722282a0fdd829359fe0d7e22adb882Chris Lattner assert((!EnableFastISelAbort || EnableFastISel) && 196b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff "-fast-isel-abort requires -fast-isel"); 197b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff 198b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff const Function &Fn = *mf.getFunction(); 199b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff const TargetInstrInfo &TII = *TM.getInstrInfo(); 200b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); 201b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff 202b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff MF = &mf; 203b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff RegInfo = &MF->getRegInfo(); 204b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff AA = &getAnalysis<AliasAnalysis>(); 205b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : 0; 206b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff 207b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); 208e4f2142d00fa5fdb580c4e2413da91882d955381Chris Lattner 209b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff CurDAG->init(*MF); 210b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff FuncInfo->set(Fn, *MF, EnableFastISel); 211b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff SDB->init(GFI, *AA); 212d7407dc92c7d19cafce429e7e1cf9819d3fc0b92Daniel Dunbar 213b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff SelectAllBasicBlocks(Fn); 214b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff 215b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // If the first basic block in the function has live ins that need to be 216b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // copied into vregs, emit the copies into the top of the block before 217b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // emitting the code for the block. 218b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff MachineBasicBlock *EntryMBB = MF->begin(); 219b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff RegInfo->EmitLiveInCopies(EntryMBB, TRI, TII); 220b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff 221b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // Insert DBG_VALUE instructions for function arguments to the entry block. 222d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) { 223ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; 224aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner unsigned Reg = MI->getOperand(0).getReg(); 225d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (TargetRegisterInfo::isPhysicalRegister(Reg)) 226f3dd57e5378bcf5fc9f05832e92479a535d9cd8aChris Lattner EntryMBB->insert(EntryMBB->begin(), MI); 227f3dd57e5378bcf5fc9f05832e92479a535d9cd8aChris Lattner else { 2281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump MachineInstr *Def = RegInfo->getVRegDef(Reg); 229f3dd57e5378bcf5fc9f05832e92479a535d9cd8aChris Lattner MachineBasicBlock::iterator InsertPos = Def; 230f3dd57e5378bcf5fc9f05832e92479a535d9cd8aChris Lattner EntryMBB->insert(llvm::next(InsertPos), MI); 2311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 232aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner } 233aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner 234aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner // Release function-specific state. SDB and CurDAG are already cleared 235aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner // at this point. 2361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump FuncInfo->clear(); 237aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner 238aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner return true; 239f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner} 240aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner 241d999b3736ce2646ae0711416b421d906298764dbBenjamin KramerMachineBasicBlock * 242aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris LattnerSelectionDAGISel::SelectBasicBlock(MachineBasicBlock *BB, 243d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer const BasicBlock *LLVMBB, 244aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner BasicBlock::const_iterator Begin, 245aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner BasicBlock::const_iterator End, 2461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump bool &HadTailCall) { 247aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner // Lower all of the non-terminator instructions. If a call is emitted 248aadaf78d65daef3ac1b45e4ad6136ce859962fe2Chris Lattner // as a tail call, cease emitting nodes for this block. Terminators 2491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // are handled below. 250f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) 251ab972d3c4b49fb0b99ae2beee70c1771a05342baSteve Naroff SDB->visit(*I); 252452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian 253f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // Make sure the root of the DAG is up-to-date. 254a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek CurDAG->setRoot(SDB->getControlRoot()); 255a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff HadTailCall = SDB->HasTailCall; 256a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff SDB->clear(); 257a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff 258a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek // Final step, emit the lowered DAG as machine code. 2594afa39deaa245592977136d367251ee2c173dd8dDouglas Gregor return CodeGenAndEmitDAG(BB); 260a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek} 2617c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 2627c63fddad600e710042fa018768807bd04eaa233Fariborz Jahaniannamespace { 263a73165e47aefbea60312d284343660c9c962c9c3Fariborz Jahanian/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted 264a73165e47aefbea60312d284343660c9c962c9c3Fariborz Jahanian/// nodes from the worklist. 265a526c5c67e5a0473c340903ee542ce570119665fTed Kremenekclass SDOPsWorkListRemover : public SelectionDAG::DAGUpdateListener { 266a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek SmallVector<SDNode*, 128> &Worklist; 267a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek SmallPtrSet<SDNode*, 128> &InWorklist; 268a526c5c67e5a0473c340903ee542ce570119665fTed Kremenekpublic: 2696327e0d55c590b3c2766fa76ef1db241a0467df2Steve Naroff SDOPsWorkListRemover(SmallVector<SDNode*, 128> &wl, 27009b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff SmallPtrSet<SDNode*, 128> &inwl) 271fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar : Worklist(wl), InWorklist(inwl) {} 272fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar 273abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian void RemoveFromWorklist(SDNode *N) { 274a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (!InWorklist.erase(N)) return; 2754c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 2764f95b750534f2111f28434b282bcbd5656002816Steve Naroff SmallVector<SDNode*, 128>::iterator I = 277d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff std::find(Worklist.begin(), Worklist.end(), N); 278d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian assert(I != Worklist.end() && "Not in worklist"); 279874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff 280d82a9ab4540899e24c96a389c5488381c5551c78Steve Naroff *I = Worklist.back(); 2811f90622e9d24064164df1608ea125d0ed451ac68Fariborz Jahanian Worklist.pop_back(); 282baf58c3f1fb13eec46fe339287662f180bb566daSteve Naroff } 2831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 284f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner virtual void NodeDeleted(SDNode *N, SDNode *E) { 285f3473a7e48dfaaa88f58dd9304856e16754f9b1cSteve Naroff RemoveFromWorklist(N); 286c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff } 2871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2888599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff virtual void NodeUpdated(SDNode *N) { 2898599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // Ignore updates. 2901eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 291e64b7771dca65d737dfc49b6137bd69fc1ff6bd2Chris Lattner}; 2922b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian} 2932b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian 2942b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian/// TrivialTruncElim - Eliminate some trivial nops that can result from 295c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff/// ShrinkDemandedOps: (trunc (ext n)) -> n. 2961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic bool TrivialTruncElim(SDValue Op, 297b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff TargetLowering::TargetLoweringOpt &TLO) { 298b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff SDValue N0 = Op.getOperand(0); 299e64b7771dca65d737dfc49b6137bd69fc1ff6bd2Chris Lattner EVT VT = Op.getValueType(); 300beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff if ((N0.getOpcode() == ISD::ZERO_EXTEND || 30136ee2cb3247a662b6049f9cc097ba5cf9c0bb2b5Fariborz Jahanian N0.getOpcode() == ISD::SIGN_EXTEND || 302b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff N0.getOpcode() == ISD::ANY_EXTEND) && 303b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff N0.getOperand(0).getValueType() == VT) { 304b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return TLO.CombineTo(Op, N0.getOperand(0)); 305b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff } 306a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek return false; 307a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian} 308a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 309a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek/// ShrinkDemandedOps - A late transformation pass that shrink expressions 310a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek/// using TargetLowering::TargetLoweringOpt::ShrinkDemandedOp. It converts 311338d1e2ced8037b71d91fb319631846917d0cedaChris Lattner/// x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free. 312338d1e2ced8037b71d91fb319631846917d0cedaChris Lattnervoid SelectionDAGISel::ShrinkDemandedOps() { 3131eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SmallVector<SDNode*, 128> Worklist; 3141d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian SmallPtrSet<SDNode*, 128> InWorklist; 3151d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian 3161d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian // Add all the dag nodes to the worklist. 3171d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian Worklist.reserve(CurDAG->allnodes_size()); 3181d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 3191d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian E = CurDAG->allnodes_end(); I != E; ++I) { 320e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian Worklist.push_back(I); 321e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian InWorklist.insert(I); 32233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian } 3231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 32409b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff TargetLowering::TargetLoweringOpt TLO(*CurDAG, true, true, true); 325874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff while (!Worklist.empty()) { 32680a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian SDNode *N = Worklist.pop_back_val(); 327acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian InWorklist.erase(N); 32880a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian 32909b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff if (N->use_empty() && N != CurDAG->getRoot().getNode()) { 3309bcb5fc1fd48c1f40c6a3b5a59130ebc313b4957Steve Naroff // Deleting this node may make its operands dead, add them to the worklist 331d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian // if they aren't already there. 332a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 333c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff if (InWorklist.insert(N->getOperand(i).getNode())) 3341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Worklist.push_back(N->getOperand(i).getNode()); 335f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 336a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek CurDAG->DeleteNode(N); 337ccd87b07a714b6e2735a80e89e425ce7fdf6a8d8Fariborz Jahanian continue; 3381eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 339a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 340ccd87b07a714b6e2735a80e89e425ce7fdf6a8d8Fariborz Jahanian // Run ShrinkDemandedOp on scalar binary operations. 3411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (N->getNumValues() != 1 || 342653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor !N->getValueType(0).isSimple() || !N->getValueType(0).isInteger()) 343653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor continue; 344653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor 3458e991ba2e256e56959a65a049e161aa30b3c0decFariborz Jahanian unsigned BitWidth = N->getValueType(0).getScalarType().getSizeInBits(); 3462e6d935930867d93d84d234ada30d93a0371754dFariborz Jahanian APInt Demanded = APInt::getAllOnesValue(BitWidth); 347158ecb9767faf87c2a33df3baec1b160dcc0be84Chris Lattner APInt KnownZero, KnownOne; 348158ecb9767faf87c2a33df3baec1b160dcc0be84Chris Lattner if (!TLI.SimplifyDemandedBits(SDValue(N, 0), Demanded, 3491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump KnownZero, KnownOne, TLO) && 350621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff (N->getOpcode() != ISD::TRUNCATE || 351621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff !TrivialTruncElim(SDValue(N, 0), TLO))) 352621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff continue; 353621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff 354621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff // Revisit the node. 3551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump assert(!InWorklist.count(N) && "Already in worklist"); 356621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff Worklist.push_back(N); 357621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff InWorklist.insert(N); 358a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 35926e4cd331c389a9b7d51c91981a7a10c8a909bf4Fariborz Jahanian // Replace the old value with the new one. 3607c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian DEBUG(errs() << "\nShrinkDemandedOps replacing "; 3611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump TLO.Old.getNode()->dump(CurDAG); 36226e4cd331c389a9b7d51c91981a7a10c8a909bf4Fariborz Jahanian errs() << "\nWith: "; 363ace6625f236af4fc81d87d63d79d7819944524dcSteve Naroff TLO.New.getNode()->dump(CurDAG); 364ace6625f236af4fc81d87d63d79d7819944524dcSteve Naroff errs() << '\n'); 3651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 36654055232a5ddb9529726e934301b125cb720a273Steve Naroff if (InWorklist.insert(TLO.New.getNode())) 3671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Worklist.push_back(TLO.New.getNode()); 36854055232a5ddb9529726e934301b125cb720a273Steve Naroff 3691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SDOPsWorkListRemover DeadNodes(Worklist, InWorklist); 37054055232a5ddb9529726e934301b125cb720a273Steve Naroff CurDAG->ReplaceAllUsesOfValueWith(TLO.Old, TLO.New, &DeadNodes); 37154055232a5ddb9529726e934301b125cb720a273Steve Naroff 3721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (!TLO.Old.getNode()->use_empty()) continue; 3731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 37454055232a5ddb9529726e934301b125cb720a273Steve Naroff for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); 37554055232a5ddb9529726e934301b125cb720a273Steve Naroff i != e; ++i) { 37652b08f2c8a65c059cb2123fa0fd6317b829416deFariborz Jahanian SDNode *OpNode = TLO.Old.getNode()->getOperand(i).getNode(); 377ab10b2ebad09c283ccab0ef043118b3cf0166b56Fariborz Jahanian if (OpNode->hasOneUse()) { 378f381cc90d6cfa0966ddf791db57a6ed6c2462b5dFariborz Jahanian // Add OpNode to the end of the list to revisit. 3796cb6eb4c792b504ad652d9230640656852e18ee9Fariborz Jahanian DeadNodes.RemoveFromWorklist(OpNode); 38054055232a5ddb9529726e934301b125cb720a273Steve Naroff Worklist.push_back(OpNode); 3811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InWorklist.insert(OpNode); 3821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 38354055232a5ddb9529726e934301b125cb720a273Steve Naroff } 3841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 38554055232a5ddb9529726e934301b125cb720a273Steve Naroff DeadNodes.RemoveFromWorklist(TLO.Old.getNode()); 38601aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff CurDAG->DeleteNode(TLO.Old.getNode()); 38701aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff } 38801aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff} 38901aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff 39001aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroffvoid SelectionDAGISel::ComputeLiveOutVRegInfo() { 39101aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff SmallPtrSet<SDNode*, 128> VisitedNodes; 3928a9e170efd32855377bc7fc5f7ea431ec4f8802eFariborz Jahanian SmallVector<SDNode*, 128> Worklist; 39354055232a5ddb9529726e934301b125cb720a273Steve Naroff 394abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian Worklist.push_back(CurDAG->getRoot().getNode()); 3953d7e7865bd0863fcf36aee14d8911b785dde57ddSteve Naroff 3961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump APInt Mask; 39754055232a5ddb9529726e934301b125cb720a273Steve Naroff APInt KnownZero; 39854055232a5ddb9529726e934301b125cb720a273Steve Naroff APInt KnownOne; 3995e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian 4005e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian do { 40172952fc11f80c975492a2a1e0f6e3601c5252e0aFariborz Jahanian SDNode *N = Worklist.pop_back_val(); 4021eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 40354055232a5ddb9529726e934301b125cb720a273Steve Naroff // If we've already seen this node, ignore it. 40454055232a5ddb9529726e934301b125cb720a273Steve Naroff if (!VisitedNodes.insert(N)) 4058189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek continue; 4068189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek 4078189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek // Otherwise, add all chain operands to the worklist. 4081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 4094fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian if (N->getOperand(i).getValueType() == MVT::Other) 4104fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian Worklist.push_back(N->getOperand(i).getNode()); 4114fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian 4124fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian // If this is a CopyToReg with a vreg dest, process it. 4134fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian if (N->getOpcode() != ISD::CopyToReg) 4144fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian continue; 4154fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian 4164fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 4174fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian if (!TargetRegisterInfo::isVirtualRegister(DestReg)) 4184fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian continue; 4194fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian 4204fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian // Ignore non-scalar or non-integer values. 42154055232a5ddb9529726e934301b125cb720a273Steve Naroff SDValue Src = N->getOperand(2); 42254055232a5ddb9529726e934301b125cb720a273Steve Naroff EVT SrcVT = Src.getValueType(); 42354055232a5ddb9529726e934301b125cb720a273Steve Naroff if (!SrcVT.isInteger() || SrcVT.isVector()) 42454055232a5ddb9529726e934301b125cb720a273Steve Naroff continue; 4251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 42654055232a5ddb9529726e934301b125cb720a273Steve Naroff unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src); 4271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits()); 42854055232a5ddb9529726e934301b125cb720a273Steve Naroff CurDAG->ComputeMaskedBits(Src, Mask, KnownZero, KnownOne); 42954055232a5ddb9529726e934301b125cb720a273Steve Naroff 43054055232a5ddb9529726e934301b125cb720a273Steve Naroff // Only install this information if it tells us something. 4311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) { 4326217b80b7a1379b74cced1c076338262c3c980b3Ted Kremenek DestReg -= TargetRegisterInfo::FirstVirtualRegister; 4331eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (DestReg >= FuncInfo->LiveOutRegInfo.size()) 434d1b3c2dd5bc1f3103bee6137957aa7c5f8f2f0bcSteve Naroff FuncInfo->LiveOutRegInfo.resize(DestReg+1); 43554055232a5ddb9529726e934301b125cb720a273Steve Naroff FunctionLoweringInfo::LiveOutInfo &LOI = 43654055232a5ddb9529726e934301b125cb720a273Steve Naroff FuncInfo->LiveOutRegInfo[DestReg]; 43754055232a5ddb9529726e934301b125cb720a273Steve Naroff LOI.NumSignBits = NumSignBits; 43854055232a5ddb9529726e934301b125cb720a273Steve Naroff LOI.KnownOne = KnownOne; 43954055232a5ddb9529726e934301b125cb720a273Steve Naroff LOI.KnownZero = KnownZero; 4408189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } 4418189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } while (!Worklist.empty()); 442b2f9e516327310d95840d442416084508f80c183Steve Naroff} 4431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 444fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve NaroffMachineBasicBlock *SelectionDAGISel::CodeGenAndEmitDAG(MachineBasicBlock *BB) { 4455e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian std::string GroupName; 4465e49b2f3e0bbc583076fe8af00dff06bcba06dafFariborz Jahanian if (TimePassesIsEnabled) 4471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump GroupName = "Instruction Selection and Scheduling"; 448621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff std::string BlockName; 4491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs || 450621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs || 451621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff ViewSUnitDAGs) 452621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff BlockName = MF->getFunction()->getNameStr() + ":" + 453621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff BB->getBasicBlock()->getNameStr(); 454621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff 455621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff DEBUG(dbgs() << "Initial selection DAG:\n"); 45677cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner DEBUG(CurDAG->dump()); 4579d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall 4589d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall if (ViewDAGCombine1) CurDAG->viewGraph("dag-combine1 input for " + BlockName); 4599d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall 4609d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall // Run the DAG combiner in pre-legalize mode. 4619d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall if (TimePassesIsEnabled) { 46241b2dcd465f1e438502c420effc9d0c747f9db8fAnders Carlsson NamedRegionTimer T("DAG Combining 1", GroupName); 4639d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall CurDAG->Combine(Unrestricted, *AA, OptLevel); 4649d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall } else { 46577cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner CurDAG->Combine(Unrestricted, *AA, OptLevel); 46677cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner } 4671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump DEBUG(dbgs() << "Optimized lowered selection DAG:\n"); 46972564e73277e29f6db3305d1f27ba408abb7ed88Douglas Gregor DEBUG(CurDAG->dump()); 4701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 47154055232a5ddb9529726e934301b125cb720a273Steve Naroff // Second step, hack on the DAG until it only uses operations and types that 47201f2ffacc427de6bef08fe138e8cae82ba1b30a3Steve Naroff // the target supports. 47354055232a5ddb9529726e934301b125cb720a273Steve Naroff if (ViewLegalizeTypesDAGs) CurDAG->viewGraph("legalize-types input for " + 47454055232a5ddb9529726e934301b125cb720a273Steve Naroff BlockName); 47554055232a5ddb9529726e934301b125cb720a273Steve Naroff 47654055232a5ddb9529726e934301b125cb720a273Steve Naroff bool Changed; 47754055232a5ddb9529726e934301b125cb720a273Steve Naroff if (TimePassesIsEnabled) { 47854055232a5ddb9529726e934301b125cb720a273Steve Naroff NamedRegionTimer T("Type Legalization", GroupName); 47954055232a5ddb9529726e934301b125cb720a273Steve Naroff Changed = CurDAG->LegalizeTypes(); 48054055232a5ddb9529726e934301b125cb720a273Steve Naroff } else { 4816217b80b7a1379b74cced1c076338262c3c980b3Ted Kremenek Changed = CurDAG->LegalizeTypes(); 48254055232a5ddb9529726e934301b125cb720a273Steve Naroff } 48372564e73277e29f6db3305d1f27ba408abb7ed88Douglas Gregor 48454055232a5ddb9529726e934301b125cb720a273Steve Naroff DEBUG(dbgs() << "Type-legalized selection DAG:\n"); 48554055232a5ddb9529726e934301b125cb720a273Steve Naroff DEBUG(CurDAG->dump()); 486b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian 487b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian if (Changed) { 4881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (ViewDAGCombineLT) 489b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian CurDAG->viewGraph("dag-combine-lt input for " + BlockName); 490b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian 4911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // Run the DAG combiner in post-type-legalize mode. 492b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian if (TimePassesIsEnabled) { 4931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NamedRegionTimer T("DAG Combining after legalize types", GroupName); 494b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian CurDAG->Combine(NoIllegalTypes, *AA, OptLevel); 495b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian } else { 496b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian CurDAG->Combine(NoIllegalTypes, *AA, OptLevel); 497b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian } 4981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 499b4b2f0c9e80c6613c3e9846da121632ea39bcd5dFariborz Jahanian DEBUG(dbgs() << "Optimized type-legalized selection DAG:\n"); 50066d6f048471aad6daf67b3ea804b54fbea72a076Eli Friedman DEBUG(CurDAG->dump()); 501c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman } 502c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman 503c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman if (TimePassesIsEnabled) { 504c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman NamedRegionTimer T("Vector Legalization", GroupName); 505a7b402dc258bf38ab5e206dbf4916a69d3ee3cc8Steve Naroff Changed = CurDAG->LegalizeVectors(); 5061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 507a7b402dc258bf38ab5e206dbf4916a69d3ee3cc8Steve Naroff Changed = CurDAG->LegalizeVectors(); 5081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 5098189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek 5108189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek if (Changed) { 511a7b402dc258bf38ab5e206dbf4916a69d3ee3cc8Steve Naroff if (TimePassesIsEnabled) { 512a7b402dc258bf38ab5e206dbf4916a69d3ee3cc8Steve Naroff NamedRegionTimer T("Type Legalization 2", GroupName); 513bce831b1ecda51cd1abd3c07467294ffd21f5f86Eli Friedman CurDAG->LegalizeTypes(); 514bce831b1ecda51cd1abd3c07467294ffd21f5f86Eli Friedman } else { 5151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump CurDAG->LegalizeTypes(); 516c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman } 517c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman 518c6d656e2b0491f224ddd48507627b51d630d749aEli Friedman if (ViewDAGCombineLT) 519e365c50a21cb02e0219433db0c4461566ad4a597Chris Lattner CurDAG->viewGraph("dag-combine-lv input for " + BlockName); 52077cd2a0b6eea81cc393b4c9e2941ec31fa09fdbeChris Lattner 521b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff // Run the DAG combiner in post-type-legalize mode. 5229e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner if (TimePassesIsEnabled) { 5239e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner NamedRegionTimer T("DAG Combining after legalize vectors", GroupName); 524ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); 5259e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner } else { 5269e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); 5279e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner } 5289e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner 5299e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner DEBUG(dbgs() << "Optimized vector-legalized selection DAG:\n"); 5309e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner DEBUG(CurDAG->dump()); 5319e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner } 532d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian 5339e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner if (ViewLegalizeDAGs) CurDAG->viewGraph("legalize input for " + BlockName); 5349e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner 5359e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner if (TimePassesIsEnabled) { 5369e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner NamedRegionTimer T("DAG Legalization", GroupName); 53754055232a5ddb9529726e934301b125cb720a273Steve Naroff CurDAG->Legalize(OptLevel); 53854055232a5ddb9529726e934301b125cb720a273Steve Naroff } else { 539abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian CurDAG->Legalize(OptLevel); 540b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff } 5419e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner 542621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff DEBUG(dbgs() << "Legalized selection DAG:\n"); 5439630ec5b6785e5b6e1c71aeddcad05005beb7483Steve Naroff DEBUG(CurDAG->dump()); 5449e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner 545c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff if (ViewDAGCombine2) CurDAG->viewGraph("dag-combine2 input for " + BlockName); 546d82a9ab4540899e24c96a389c5488381c5551c78Steve Naroff 54768272b86b36f89d7ceba1ec9a2826c2126adc72eSteve Naroff // Run the DAG combiner in post-legalize mode. 54868272b86b36f89d7ceba1ec9a2826c2126adc72eSteve Naroff if (TimePassesIsEnabled) { 549b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff NamedRegionTimer T("DAG Combining 2", GroupName); 550f292fcf6bd5118499a830e0950429effeb373c28Fariborz Jahanian CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); 5511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 5529e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); 5539e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner } 5549e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner 5559e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner DEBUG(dbgs() << "Optimized legalized selection DAG:\n"); 5569e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner DEBUG(CurDAG->dump()); 5571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5582c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner if (OptLevel != CodeGenOpt::None) { 5591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump ShrinkDemandedOps(); 5609e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner ComputeLiveOutVRegInfo(); 5619e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner } 562ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 56362c2632c555bc701d41f0eb1803bbf0660d7f0c0Steve Naroff if (ViewISelDAGs) CurDAG->viewGraph("isel input for " + BlockName); 564ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 56546a98a7df016b4a6835a1902c2f8ff2a3a8012baSteve Naroff // Third, instruction select all of the operations to machine code, adding the 566ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // code to the MachineBasicBlock. 567c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff if (TimePassesIsEnabled) { 568c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff NamedRegionTimer T("Instruction Selection", GroupName); 5698189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek DoInstructionSelection(); 5708189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } else { 571ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff DoInstructionSelection(); 572c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff } 573ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 574ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff DEBUG(dbgs() << "Selected selection DAG:\n"); 575ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff DEBUG(CurDAG->dump()); 576ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 577ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff if (ViewSchedDAGs) CurDAG->viewGraph("scheduler input for " + BlockName); 5784ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff 5794ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff // Schedule machine code. 5804ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff ScheduleDAGSDNodes *Scheduler = CreateScheduler(); 5814ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff if (TimePassesIsEnabled) { 5827c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian NamedRegionTimer T("Instruction Scheduling", GroupName); 5834ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff Scheduler->Run(CurDAG, BB, BB->end()); 584ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff } else { 5854ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff Scheduler->Run(CurDAG, BB, BB->end()); 586ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff } 5874ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff 588ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff if (ViewSUnitDAGs) Scheduler->viewGraph(); 5894ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff 590ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // Emit machine code to BB. This can change 'BB' to the last block being 5914ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff // inserted into. 592ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff if (TimePassesIsEnabled) { 5934ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff NamedRegionTimer T("Instruction Creation", GroupName); 594ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff BB = Scheduler->EmitSchedule(); 595d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian } else { 596d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian BB = Scheduler->EmitSchedule(); 5974ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff } 598ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 5994ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff // Free the scheduler state. 6004ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff if (TimePassesIsEnabled) { 6014ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName); 6024ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff delete Scheduler; 6034ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff } else { 604580ca7859330f8c92a66b09c21399ed9efa3991eSteve Naroff delete Scheduler; 60559f05a48a7b0353090da2b8f878c2d760e55731cSteve Naroff } 6064ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff 6074ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff // Free the SelectionDAG state, now that we're finished with it. 6084ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff CurDAG->clear(); 609ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 610ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff return BB; 611ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff} 612b10f273eb60db2f2205b955fe4f7a43e09a5e2f2Steve Naroff 613ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroffvoid SelectionDAGISel::DoInstructionSelection() { 614ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff DEBUG(errs() << "===== Instruction selection begins:\n"); 6154ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff 616ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff PreprocessISelDAG(); 617ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 618ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // Select target instructions for the DAG. 619ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff { 620ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // Number all nodes with a topological order and set DAGSize. 621ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff DAGSize = CurDAG->AssignTopologicalOrder(); 622ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff 623ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // Create a dummy node (which is not added to allnodes), that adds 624ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // a reference to the root node, preventing it from being deleted, 62588bee7451399c04b7e1c49cdeb16fadf0dd66f58Steve Naroff // and tracking any changes of the root. 62688bee7451399c04b7e1c49cdeb16fadf0dd66f58Steve Naroff HandleSDNode Dummy(CurDAG->getRoot()); 62788bee7451399c04b7e1c49cdeb16fadf0dd66f58Steve Naroff ISelPosition = SelectionDAG::allnodes_iterator(CurDAG->getRoot().getNode()); 6284ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff ++ISelPosition; 62988bee7451399c04b7e1c49cdeb16fadf0dd66f58Steve Naroff 630ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // The AllNodes list is now topological-sorted. Visit the 631ba92b2ed976e29ea05f0f5afabaf9488c1096bebSteve Naroff // nodes by starting at the end of the list (the root of the 63254055232a5ddb9529726e934301b125cb720a273Steve Naroff // graph) and preceding back toward the beginning (the entry 63354055232a5ddb9529726e934301b125cb720a273Steve Naroff // node). 63454055232a5ddb9529726e934301b125cb720a273Steve Naroff while (ISelPosition != CurDAG->allnodes_begin()) { 63554055232a5ddb9529726e934301b125cb720a273Steve Naroff SDNode *Node = --ISelPosition; 63654055232a5ddb9529726e934301b125cb720a273Steve Naroff // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes, 63754055232a5ddb9529726e934301b125cb720a273Steve Naroff // but there are currently some corner cases that it misses. Also, this 63801aec11c8cace2b9b22531627970d8bbdbac0c1cSteve Naroff // makes it theoretically possible to disable the DAGCombiner. 63954055232a5ddb9529726e934301b125cb720a273Steve Naroff if (Node->use_empty()) 64054055232a5ddb9529726e934301b125cb720a273Steve Naroff continue; 6415bc60d0c324a1b3984cd4a32acb587c8f873ab5fSteve Naroff 642c9c1e9c4b513e83146eff1728142683980df26a7Steve Naroff SDNode *ResNode = Select(Node); 6437c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 6447c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // FIXME: This is pretty gross. 'Select' should be changed to not return 645a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff // anything at all and this code should be nuked with a tactical strike. 646a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff 647a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff // If node should not be replaced, continue with the next one. 648a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff if (ResNode == Node || Node->getOpcode() == ISD::DELETED_NODE) 649cd826370bb146a25bddea48399953dd3077710cfSteve Naroff continue; 650cd826370bb146a25bddea48399953dd3077710cfSteve Naroff // Replace node. 651a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff if (ResNode) 652a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff ReplaceUses(Node, ResNode); 653a851e60b7eb1d51549222ac838b8b602f23ec701Steve Naroff 65454055232a5ddb9529726e934301b125cb720a273Steve Naroff // If after the replacement this node is not used any more, 655a48396e10f75966ed5ca974dcc73907ce8c80deeSteve Naroff // remove this dead node. 6564ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff if (Node->use_empty()) { // Don't delete EntryToken, etc. 6574ebd716f260584663297bdb820714788f2a1e4cbSteve Naroff ISelUpdater ISU(ISelPosition); 658553e583e1d30fe30ac2de3526536bbebee6da1ceChris Lattner CurDAG->RemoveDeadNode(Node, &ISU); 659a48396e10f75966ed5ca974dcc73907ce8c80deeSteve Naroff } 660553e583e1d30fe30ac2de3526536bbebee6da1ceChris Lattner } 66134204195de802e9d3cc8d54355b2ad4b3c27f188Fariborz Jahanian 662a48396e10f75966ed5ca974dcc73907ce8c80deeSteve Naroff CurDAG->setRoot(Dummy.getValue()); 6632086d544d38f2611f2fba034e25833a47c81fb21Fariborz Jahanian } 66452b08f2c8a65c059cb2123fa0fd6317b829416deFariborz Jahanian DEBUG(errs() << "===== Instruction selection ends:\n"); 6652086d544d38f2611f2fba034e25833a47c81fb21Fariborz Jahanian 6662086d544d38f2611f2fba034e25833a47c81fb21Fariborz Jahanian PostprocessISelDAG(); 667df496529b7009d6e58f335926884200b38d27687Fariborz Jahanian} 668df496529b7009d6e58f335926884200b38d27687Fariborz Jahanian 669df496529b7009d6e58f335926884200b38d27687Fariborz Jahanian/// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and 6709e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner/// do other setup for EH landing-pad blocks. 6719e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattnervoid SelectionDAGISel::PrepareEHLandingPad(MachineBasicBlock *BB) { 6729e13c2eb681b5d8e14f930b14300caaae22fdb92Chris Lattner // Add a label to mark the beginning of the landing pad. Deletion of the 673f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // landing pad can thus be detected via the MachineModuleInfo. 674f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner MCSymbol *Label = MF->getMMI().addLandingPad(BB); 675f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 676f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner const TargetInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); 677682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattner BuildMI(BB, SDB->getCurDebugLoc(), II).addSym(Label); 678e50187a987dadb6a3b6f673125617c8f42ff3560Ted Kremenek 679e50187a987dadb6a3b6f673125617c8f42ff3560Ted Kremenek // Mark exception register as live in. 680e50187a987dadb6a3b6f673125617c8f42ff3560Ted Kremenek unsigned Reg = TLI.getExceptionAddressRegister(); 6812c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner if (Reg) BB->addLiveIn(Reg); 6822c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner 6832c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner // Mark exception selector register as live in. 6842c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner Reg = TLI.getExceptionSelectorRegister(); 685f7cf85b330bedd2877e1371fb0a83e99751ae162Chris Lattner if (Reg) BB->addLiveIn(Reg); 6861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6872c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner // FIXME: Hack around an exception handling flaw (PR1508): the personality 6882c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner // function and list of typeids logically belong to the invoke (or, if you 6892c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner // like, the basic block containing the invoke), and need to be associated 690ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff // with it in the dwarf exception handling tables. Currently however the 691ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff // information is provided by an intrinsic (eh.selector) that can be moved 69209b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff // to unexpected places by the optimizers: if the unwind edge is critical, 693248a753f6b670692523c99afaeb8fe98f7ae3ca7Steve Naroff // then breaking it can result in the intrinsics being in the successor of 694beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff // the landing pad, not the landing pad itself. This results 6958ec03f58c33c33a917f54bb7f2cd61b6d7ffe0caChris Lattner // in exceptions not being caught because no typeids are associated with 696beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff // the invoke. This may not be the only way things can go wrong, but it 697beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff // is the only way we try to work around for the moment. 698beaf299a2701c5559a4e5d76b0c40f805afb8e6aSteve Naroff const BasicBlock *LLVMBB = BB->getBasicBlock(); 699a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek const BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator()); 700bef1185418705e16012b3dd50cd7c260c8d6b79cSteve Naroff 701a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (Br && Br->isUnconditional()) { // Critical edge? 702423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff BasicBlock::const_iterator I, E; 703a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I) 704752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff if (isa<EHSelectorInst>(I)) 7051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump break; 706a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 707d175ddfdbefff8646982dfd4afc5bdf6edbc9e67Fariborz Jahanian if (I == E) 708d04341000d35c8808a72838b057eed7bf13b7661Douglas Gregor // No catch info found - try to extract some from the successor. 709d04341000d35c8808a72838b057eed7bf13b7661Douglas Gregor CopyCatchInfo(Br->getSuccessor(0), LLVMBB, &MF->getMMI(), *FuncInfo); 71017945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis } 71117945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis} 712d04341000d35c8808a72838b057eed7bf13b7661Douglas Gregor 713682bf92db408a6cbc3d37b5496a99b6ef85041ecChris Lattnervoid SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { 714ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff // Initialize the Fast-ISel state, if needed. 715f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner FastISel *FastIS = 0; 716cf7e958483735935de6e0a64aa1e55a967116303Ted Kremenek if (EnableFastISel) 7172c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner FastIS = TLI.createFastISel(*MF, FuncInfo->ValueMap, FuncInfo->MBBMap, 7182c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner FuncInfo->StaticAllocaMap, 7192c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner FuncInfo->PHINodesToUpdate 720f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner#ifndef NDEBUG 721f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner , FuncInfo->CatchInfoLost 722f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner#endif 723f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner ); 724b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff 7252b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // Iterate over all basic blocks in the function. 726f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer for (Function::const_iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { 727f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer const BasicBlock *LLVMBB = &*I; 728f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer MachineBasicBlock *BB = FuncInfo->MBBMap[LLVMBB]; 729452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian 7301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump BasicBlock::const_iterator const Begin = LLVMBB->getFirstNonPHI(); 731af57b46f12e9dd48485fa27f2088f85466ee9292Fariborz Jahanian BasicBlock::const_iterator const End = LLVMBB->end(); 732452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian BasicBlock::const_iterator BI = Begin; 733452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian 734452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian // Lower any arguments needed in this block if this is the entry block. 735452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian if (LLVMBB == &Fn.getEntryBlock()) 736452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian LowerArguments(LLVMBB); 737452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian 738452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian // Setup an EH landing-pad block. 739452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian if (BB->isLandingPad()) 740452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian PrepareEHLandingPad(BB); 7411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 742452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian // Before doing SelectionDAG ISel, see if FastISel has been requested. 743d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (FastIS) { 744452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian // Emit code for any incoming arguments. This must happen before 745452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian // beginning FastISel on the entry block. 746452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian if (LLVMBB == &Fn.getEntryBlock()) { 747452b899c9f5fbcb8ddd130375b7982bed6b4d93fFariborz Jahanian CurDAG->setRoot(SDB->getControlRoot()); 7488a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner SDB->clear(); 7498a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner BB = CodeGenAndEmitDAG(BB); 750b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff } 751f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer FastIS->startNewBlock(BB); 752f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer // Do FastISel on as many instructions as possible. 753f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer for (; BI != End; ++BI) { 7541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // Try to select the instruction with FastISel. 755f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner if (FastIS->SelectInstruction(BI)) 756f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner continue; 757f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 758f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // Then handle certain instructions as single-LLVM-Instruction blocks. 7591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (isa<CallInst>(BI)) { 760f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner ++NumFastIselFailures; 761f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner if (EnableFastISelVerbose || EnableFastISelAbort) { 762f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner dbgs() << "FastISel missed call: "; 763f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner BI->dump(); 764f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner } 765f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 7661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (!BI->getType()->isVoidTy() && !BI->use_empty()) { 767f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner unsigned &R = FuncInfo->ValueMap[BI]; 768f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner if (!R) 769f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner R = FuncInfo->CreateRegForValue(BI); 7701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 771f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 7722b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner bool HadTailCall = false; 7732b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner BB = SelectBasicBlock(BB, LLVMBB, BI, llvm::next(BI), HadTailCall); 7741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 775f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // If the call was emitted as a tail call, we're done with the block. 776d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (HadTailCall) { 777f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner BI = End; 7782c64b7b9381be4ff62fbdc404ed3f14c8086898dChris Lattner break; 7798a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner } 780eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff 781eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // If the instruction was codegen'd with multiple blocks, 782eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // inform the FastISel object where to resume inserting. 783eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff FastIS->setCurrentBlock(BB); 784eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff continue; 785eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff } 7865ffe14ca96bd662de7820f6875d3f04789a640c1Daniel Dunbar 787eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // Otherwise, give up on FastISel for the rest of the block. 788eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // For now, be a little lenient about non-branch terminators. 789eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff if (!isa<TerminatorInst>(BI) || isa<BranchInst>(BI)) { 790a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff ++NumFastIselFailures; 791a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff if (EnableFastISelVerbose || EnableFastISelAbort) { 792a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff dbgs() << "FastISel miss: "; 7937c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BI->dump(); 7947c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 795d40910b581b09c937a8c1fdcde9b8ec724398fb9Steve Naroff if (EnableFastISelAbort) 796d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // The "fast" selector couldn't handle something and bailed. 797eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // For the purpose of debugging, just abort. 798eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff llvm_unreachable("FastISel didn't select the entire block"); 799eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff } 800eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff break; 8018189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } 8028189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } 803eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff 804eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // Run SelectionDAG instruction selection on the remainder of the block 805eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // not handled by FastISel. If FastISel is not run, this is the entire 8061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // block. 807eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff if (BI != End) { 808eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff bool HadTailCall; 809eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff BB = SelectBasicBlock(BB, LLVMBB, BI, End, HadTailCall); 810eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff } 8111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 812dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff FinishBasicBlock(BB); 813dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff FuncInfo->PHINodesToUpdate.clear(); 8147c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 8157c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8167c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian delete FastIS; 8177c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian} 818dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff 8197c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanianvoid 8207c63fddad600e710042fa018768807bd04eaa233Fariborz JahanianSelectionDAGISel::FinishBasicBlock(MachineBasicBlock *BB) { 8217c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8227c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian DEBUG(dbgs() << "Total amount of phi nodes to update: " 8237c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian << FuncInfo->PHINodesToUpdate.size() << "\n"); 8247c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian DEBUG(for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) 825dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff dbgs() << "Node " << i << " : (" 826dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff << FuncInfo->PHINodesToUpdate[i].first 827dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n"); 8283539cdb98b1132d30795f932920ff1c90f94b4e8Steve Naroff 8297c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Next, now that we know what the last MBB the LLVM BB expanded is, update 8307c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // PHI nodes in successors. 8317c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (SDB->SwitchCases.empty() && 8327c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->JTCases.empty() && 8337c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->BitTestCases.empty()) { 8347c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { 8357c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; 8367c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian assert(PHI->isPHI() && 8377c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian "This is not a machine PHI node that we are updating!"); 8387c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (!BB->isSuccessor(PHI->getParent())) 8397c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian continue; 840fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar PHI->addOperand( 8417c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); 8427c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian PHI->addOperand(MachineOperand::CreateMBB(BB)); 8437c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 844fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar return; 845fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar } 8467c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8477c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian for (unsigned i = 0, e = SDB->BitTestCases.size(); i != e; ++i) { 8487c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Lower header first, if it wasn't already lowered 8497c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (!SDB->BitTestCases[i].Emitted) { 8507c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Set the current basic block to the mbb we wish to insert the code into 8517c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BB = SDB->BitTestCases[i].Parent; 8527c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Emit the code 8537c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->visitBitTestHeader(SDB->BitTestCases[i], BB); 8547c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian CurDAG->setRoot(SDB->getRoot()); 8557c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->clear(); 8567c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BB = CodeGenAndEmitDAG(BB); 8577c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 8587c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8597c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { 8607c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Set the current basic block to the mbb we wish to insert the code into 8617c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BB = SDB->BitTestCases[i].Cases[j].ThisBB; 8627c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // Emit the code 8637c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (j+1 != ej) 864dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff SDB->visitBitTestCase(SDB->BitTestCases[i].Cases[j+1].ThisBB, 865d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer SDB->BitTestCases[i].Reg, 866eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff SDB->BitTestCases[i].Cases[j], 867eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff BB); 8681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump else 869eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff SDB->visitBitTestCase(SDB->BitTestCases[i].Default, 870eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff SDB->BitTestCases[i].Reg, 8717c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->BitTestCases[i].Cases[j], 8727c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BB); 8737c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8747c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 8757c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian CurDAG->setRoot(SDB->getRoot()); 8767c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian SDB->clear(); 8777c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian BB = CodeGenAndEmitDAG(BB); 8787c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 8797c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 880eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff // Update PHI Nodes 881eb0646c8df1c5c8de62b622d13d0c9f19aa29277Steve Naroff for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); 882dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff pi != pe; ++pi) { 88315f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; 8847c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian MachineBasicBlock *PHIBB = PHI->getParent(); 8857c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian assert(PHI->isPHI() && 8867c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian "This is not a machine PHI node that we are updating!"); 8877c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // This is "default" BB. We have two jumps to it. From "header" BB and 8887c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // from last "case" BB. 8897c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (PHIBB == SDB->BitTestCases[i].Default) { 8907c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian PHI->addOperand(MachineOperand:: 8917c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian CreateReg(FuncInfo->PHINodesToUpdate[pi].second, 8927c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian false)); 8937c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Parent)); 8947c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian PHI->addOperand(MachineOperand:: 8957c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian CreateReg(FuncInfo->PHINodesToUpdate[pi].second, 8967c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian false)); 8977c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Cases. 8987c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian back().ThisBB)); 8997c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 9007c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // One of "cases" BB. 9017c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); 9027c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian j != ej; ++j) { 903dd2fdf13f11f93a8bd271807db25c71191914807Steve Naroff MachineBasicBlock* cBB = SDB->BitTestCases[i].Cases[j].ThisBB; 904d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (cBB->isSuccessor(PHIBB)) { 905d40910b581b09c937a8c1fdcde9b8ec724398fb9Steve Naroff PHI->addOperand(MachineOperand:: 9068a12c2777cccdf629b89745b6ecc89a8c1641e4eChris Lattner CreateReg(FuncInfo->PHINodesToUpdate[pi].second, 907b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff false)); 908f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner PHI->addOperand(MachineOperand::CreateMBB(cBB)); 909f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner } 910f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner } 911f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner } 9121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 913f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner SDB->BitTestCases.clear(); 914f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner 915f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // If the JumpTable record is filled in, then we need to emit a jump table. 916f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner // Updating the PHI nodes is tricky in this case, since we need to determine 91791fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian // whether the PHI is a successor of the range check MBB or the jump table MBB 91891fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) { 91991fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian // Lower header first, if it wasn't already lowered 92091fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian if (!SDB->JTCases[i].first.Emitted) { 92191fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian // Set the current basic block to the mbb we wish to insert the code into 92291fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian BB = SDB->JTCases[i].first.HeaderBB; 92391fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian // Emit the code 92491fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first, 92591fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian BB); 92691fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian CurDAG->setRoot(SDB->getRoot()); 92791fbd12d8b6e60b72883ff4c42e2a0993087f876Fariborz Jahanian SDB->clear(); 92867956052ea5fb0cd7f443de96a11f9a0176dc681Chris Lattner BB = CodeGenAndEmitDAG(BB); 92967956052ea5fb0cd7f443de96a11f9a0176dc681Chris Lattner } 930321c22f1c4271c3d9a3d4d3fc18847f948ab595bTed Kremenek 93132174826f58d38c20f376fe64ceac4a278133883Steve Naroff // Set the current basic block to the mbb we wish to insert the code into 932d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner BB = SDB->JTCases[i].second.MBB; 93332174826f58d38c20f376fe64ceac4a278133883Steve Naroff // Emit the code 93432174826f58d38c20f376fe64ceac4a278133883Steve Naroff SDB->visitJumpTable(SDB->JTCases[i].second); 935d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner CurDAG->setRoot(SDB->getRoot()); 93632174826f58d38c20f376fe64ceac4a278133883Steve Naroff SDB->clear(); 937352336bb0a480b044eb03e7359e73d3b173d9cddSteve Naroff BB = CodeGenAndEmitDAG(BB); 938d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner 93932174826f58d38c20f376fe64ceac4a278133883Steve Naroff // Update PHI Nodes 940934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroff for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); 9411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump pi != pe; ++pi) { 942934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroff MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; 943d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer MachineBasicBlock *PHIBB = PHI->getParent(); 944f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner assert(PHI->isPHI() && 945f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner "This is not a machine PHI node that we are updating!"); 946b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff // "default" BB. We can go there only from header BB. 947d0502407c1b41b2ace326f355d7b7a6876246223Fariborz Jahanian if (PHIBB == SDB->JTCases[i].second.Default) { 948d0502407c1b41b2ace326f355d7b7a6876246223Fariborz Jahanian PHI->addOperand 949d0502407c1b41b2ace326f355d7b7a6876246223Fariborz Jahanian (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, 950d0502407c1b41b2ace326f355d7b7a6876246223Fariborz Jahanian false)); 95158dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff PHI->addOperand 95258dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff (MachineOperand::CreateMBB(SDB->JTCases[i].first.HeaderBB)); 9531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 95430fc933e5fbbb5f0ea60c47976d435254e378536Chris Lattner // JT BB. Just iterate over successors here 95530fc933e5fbbb5f0ea60c47976d435254e378536Chris Lattner if (BB->isSuccessor(PHIBB)) { 956d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer PHI->addOperand 957d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, 95858dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff false)); 959d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer PHI->addOperand(MachineOperand::CreateMBB(BB)); 960423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff } 961423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff } 962423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff } 9631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SDB->JTCases.clear(); 964d0502407c1b41b2ace326f355d7b7a6876246223Fariborz Jahanian 9651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If the switch block involved a branch to one of the actual successors, we 966d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // need to update PHI nodes in that block. 9676327e0d55c590b3c2766fa76ef1db241a0467df2Steve Naroff for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { 968957cf65bf32fe7c84886a2296d0b4ede40029850Fariborz Jahanian MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; 969957cf65bf32fe7c84886a2296d0b4ede40029850Fariborz Jahanian assert(PHI->isPHI() && 970b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff "This is not a machine PHI node that we are updating!"); 971423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff if (BB->isSuccessor(PHI->getParent())) { 9721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump PHI->addOperand( 973423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); 974d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer PHI->addOperand(MachineOperand::CreateMBB(BB)); 9751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 97613751e3edc7fe50b0ab4e0f5fe7501e81c83c795Fariborz Jahanian } 97713751e3edc7fe50b0ab4e0f5fe7501e81c83c795Fariborz Jahanian 97813751e3edc7fe50b0ab4e0f5fe7501e81c83c795Fariborz Jahanian // If we generated any switch lowering information, build and codegen any 97913751e3edc7fe50b0ab4e0f5fe7501e81c83c795Fariborz Jahanian // additional DAGs necessary. 9801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) { 9811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // Set the current basic block to the mbb we wish to insert the code into 9826ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor MachineBasicBlock *ThisBB = BB = SDB->SwitchCases[i].ThisBB; 98358dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff 9841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // Determine the unique successors. 98517945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis SmallVector<MachineBasicBlock *, 2> Succs; 9866ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor Succs.push_back(SDB->SwitchCases[i].TrueBB); 98758dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB) 98858dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff Succs.push_back(SDB->SwitchCases[i].FalseBB); 989423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff 99073d1eb064350d5310f0475366cbe54d2d1da27bbFariborz Jahanian // Emit the code. Note that this could result in ThisBB being split, so 99173d1eb064350d5310f0475366cbe54d2d1da27bbFariborz Jahanian // we need to check for updates. 992423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff SDB->visitSwitchCase(SDB->SwitchCases[i], BB); 993423cb565abc681b770fb4b9b4bc24d398c98157bSteve Naroff CurDAG->setRoot(SDB->getRoot()); 994b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff SDB->clear(); 995752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff ThisBB = CodeGenAndEmitDAG(BB); 9961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 997752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff // Handle any PHI nodes in successors of this chunk, as if we were coming 998d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // from the original BB before switch expansion. Note that PHI nodes can 9991eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // occur multiple times in PHINodesToUpdate. We have to be very careful to 10001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // handle them the right number of times. 10011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump for (unsigned i = 0, e = Succs.size(); i != e; ++i) { 10026ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor BB = Succs[i]; 100358dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff // BB may have been removed from the CFG if a branch was constant folded. 10046ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor if (ThisBB->isSuccessor(BB)) { 100517945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis for (MachineBasicBlock::iterator Phi = BB->begin(); 10066ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor Phi != BB->end() && Phi->isPHI(); 100758dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff ++Phi) { 100858dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff // This value for this PHI node is recorded in PHINodesToUpdate. 1009752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff for (unsigned pn = 0; ; ++pn) { 1010782f2f52b78d8ca785110398a7f7b56b830b9ac7Ted Kremenek assert(pn != FuncInfo->PHINodesToUpdate.size() && 101173d1eb064350d5310f0475366cbe54d2d1da27bbFariborz Jahanian "Didn't find PHI entry!"); 10128cc764c67afc249628ba4f65b8e2d2ad461112ffSteve Naroff if (FuncInfo->PHINodesToUpdate[pn].first == Phi) { 1013b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian Phi->addOperand(MachineOperand:: 1014b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian CreateReg(FuncInfo->PHINodesToUpdate[pn].second, 1015b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian false)); 1016b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian Phi->addOperand(MachineOperand::CreateMBB(ThisBB)); 1017b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian break; 10188cc764c67afc249628ba4f65b8e2d2ad461112ffSteve Naroff } 1019d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer } 10201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 1021b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian } 1022b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian } 10238cc764c67afc249628ba4f65b8e2d2ad461112ffSteve Naroff } 1024d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer SDB->SwitchCases.clear(); 10251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 1026b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian 1027b82b3ea8a12e4bc5960f59c1c2b9f483af5471e8Fariborz Jahanian 1028752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff/// Create the scheduler. If a specific scheduler was specified 1029752d6ef8c886b4de3365efef15449cd6f939baf1Steve Naroff/// via the SchedulerRegistry, use it, otherwise select the 1030b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff/// one preferred by the target. 1031d175ddfdbefff8646982dfd4afc5bdf6edbc9e67Fariborz Jahanian/// 1032b7fa992be16668b61f722e6e125137216860469eSteve NaroffScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() { 1033b7fa992be16668b61f722e6e125137216860469eSteve Naroff RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault(); 1034d175ddfdbefff8646982dfd4afc5bdf6edbc9e67Fariborz Jahanian 1035d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (!Ctor) { 1036d175ddfdbefff8646982dfd4afc5bdf6edbc9e67Fariborz Jahanian Ctor = ISHeuristic; 1037d175ddfdbefff8646982dfd4afc5bdf6edbc9e67Fariborz Jahanian RegisterScheduler::setDefault(Ctor); 10387c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian } 10397c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 10407c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian return Ctor(this, OptLevel); 1041c569249ca0ab755ac79d8cbbfcb2bcae19743624Fariborz Jahanian} 10427c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 10437c63fddad600e710042fa018768807bd04eaa233Fariborz JahanianScheduleHazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { 104476e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff return new ScheduleHazardRecognizer(); 104576e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff} 10467c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 1047f4312dc9b690fc79039560c125e9e3f8ea8a478eSteve Naroff//===----------------------------------------------------------------------===// 10486217b80b7a1379b74cced1c076338262c3c980b3Ted Kremenek// Helper functions used by the generated instruction selector. 1049f4312dc9b690fc79039560c125e9e3f8ea8a478eSteve Naroff//===----------------------------------------------------------------------===// 10506217b80b7a1379b74cced1c076338262c3c980b3Ted Kremenek// Calls to these methods are generated by tblgen. 1051f4312dc9b690fc79039560c125e9e3f8ea8a478eSteve Naroff 1052183700f494ec9b6701b6efe82bcb25f4c79ba561John McCall/// CheckAndMask - The isel is trying to match something like (and X, 255). If 1053fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar/// the dag combiner simplified the 255, we still want to match. RHS is the 1054fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar/// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value 1055f4312dc9b690fc79039560c125e9e3f8ea8a478eSteve Naroff/// specified in the .td file (e.g. 255). 105676e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroffbool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS, 105776e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff int64_t DesiredMaskS) const { 1058fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar const APInt &ActualMask = RHS->getAPIntValue(); 10597c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); 10607c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 10617c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // If the actual mask exactly matches, success! 10627c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (ActualMask == DesiredMask) 10637c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian return true; 10647c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian 10657c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian // If the actual AND mask is allowing unallowed bits, this doesn't match. 10667c63fddad600e710042fa018768807bd04eaa233Fariborz Jahanian if (ActualMask.intersects(~DesiredMask)) 1067531a1ea2f4c88817f3dc0f4628dfc34536f54c5eFariborz Jahanian return false; 10681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 106948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // Otherwise, the DAG Combiner may have proven that the value coming in is 1070b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian // either already zero or is not demanded. Check for known zero input bits. 10711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump APInt NeededMask = DesiredMask & ~ActualMask; 1072f8d49f64ef6ab7e632717a31631fc289aab69428Douglas Gregor if (CurDAG->MaskedValueIsZero(LHS, NeededMask)) 1073b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian return true; 107448a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 1075b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian // TODO: check to see if missing bits are just not demanded. 10761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1077d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner // Otherwise, this pattern doesn't match. 1078b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian return false; 10791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 10801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10813e0a540b6d846178857289ec0eb8470a278d11a3Steve Naroff/// CheckOrMask - The isel is trying to match something like (or X, 255). If 1082d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner/// the dag combiner simplified the 255, we still want to match. RHS is the 1083b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian/// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value 108448a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian/// specified in the .td file (e.g. 255). 10851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpbool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS, 1086077bf5e2f48acfa9e7d69429b6e4ba86ea14896dChris Lattner int64_t DesiredMaskS) const { 1087077bf5e2f48acfa9e7d69429b6e4ba86ea14896dChris Lattner const APInt &ActualMask = RHS->getAPIntValue(); 108848a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); 108948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 109048a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // If the actual mask exactly matches, success! 109148a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian if (ActualMask == DesiredMask) 1092b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian return true; 109348a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 1094b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian // If the actual AND mask is allowing unallowed bits, this doesn't match. 1095b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian if (ActualMask.intersects(~DesiredMask)) 1096b7908b5c54aeafc1ff98797e304d61c68b1aed25Fariborz Jahanian return false; 10971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 109848a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // Otherwise, the DAG Combiner may have proven that the value coming in is 109948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // either already zero or is not demanded. Check for known zero input bits. 11001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump APInt NeededMask = DesiredMask & ~ActualMask; 110148a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 1102f8d49f64ef6ab7e632717a31631fc289aab69428Douglas Gregor APInt KnownZero, KnownOne; 1103a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek CurDAG->ComputeMaskedBits(LHS, NeededMask, KnownZero, KnownOne); 110448a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 110505b8c782669d07a71d4889bcacfd53a29cd16fcbSteve Naroff // If all the missing bits in the or are already known to be set, match! 110605b8c782669d07a71d4889bcacfd53a29cd16fcbSteve Naroff if ((NeededMask & KnownOne) == NeededMask) 110705b8c782669d07a71d4889bcacfd53a29cd16fcbSteve Naroff return true; 110805b8c782669d07a71d4889bcacfd53a29cd16fcbSteve Naroff 110905b8c782669d07a71d4889bcacfd53a29cd16fcbSteve Naroff // TODO: check to see if missing bits are just not demanded. 1110d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner 111161ed9caca5dd6a9b9ccee7fb51296234e6cc68adSteve Naroff // Otherwise, this pattern doesn't match. 111248a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian return false; 111348a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian} 1114fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar 1115fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar 11161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated 111748a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian/// by tblgen. Others should not call it. 1118fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbarvoid SelectionDAGISel:: 111948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz JahanianSelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) { 11201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump std::vector<SDValue> InOps; 112148a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian std::swap(InOps, Ops); 112289951a86b594513c2a013532ed45d197413b1087Chris Lattner 112389951a86b594513c2a013532ed45d197413b1087Chris Lattner Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0 112489951a86b594513c2a013532ed45d197413b1087Chris Lattner Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1 112548a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc 1126543409e200234a1c97ea38b9d52bce22ff807080Steve Naroff 1127543409e200234a1c97ea38b9d52bce22ff807080Steve Naroff unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size(); 1128d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner if (InOps[e-1].getValueType() == MVT::Flag) 1129543409e200234a1c97ea38b9d52bce22ff807080Steve Naroff --e; // Don't process a flag operand if it is here. 1130d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner 11314fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian while (i != e) { 11324fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue(); 11334fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian if (!InlineAsm::isMemKind(Flags)) { 11344fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian // Just skip over this operand, copying the operands verbatim. 11354fc8453dd02d015b1161d83a5740632617aedd12Fariborz Jahanian Ops.insert(Ops.end(), InOps.begin()+i, 1136d249e1d1f1498b81314459ceda19d6ff25c278adDouglas Gregor InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1); 1137543409e200234a1c97ea38b9d52bce22ff807080Steve Naroff i += InlineAsm::getNumOperandRegisters(Flags) + 1; 1138543409e200234a1c97ea38b9d52bce22ff807080Steve Naroff } else { 113948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian assert(InlineAsm::getNumOperandRegisters(Flags) == 1 && 11407c39ff7eb3dd7ae9b2b81cb66a139c9b52f47f1dFariborz Jahanian "Memory operand with multiple values?"); 11417c39ff7eb3dd7ae9b2b81cb66a139c9b52f47f1dFariborz Jahanian // Otherwise, this is a memory operand. Ask the target to select it. 1142531a1ea2f4c88817f3dc0f4628dfc34536f54c5eFariborz Jahanian std::vector<SDValue> SelOps; 11431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps)) 114476e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff report_fatal_error("Could not match memory address. Inline asm" 114576e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff " failure!"); 11461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 114776e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff // Add this to the output node. 114872564e73277e29f6db3305d1f27ba408abb7ed88Douglas Gregor unsigned NewFlags = 114976e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size()); 115076e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff Ops.push_back(CurDAG->getTargetConstant(NewFlags, MVT::i32)); 115176e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff Ops.insert(Ops.end(), SelOps.begin(), SelOps.end()); 1152fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar i += 2; 1153fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar } 115476e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff } 115576e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff 115676e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff // Add the flag input back if present. 115776e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff if (e != InOps.size()) 115876e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff Ops.push_back(InOps.back()); 115976e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff} 116076e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff 116176e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff/// findFlagUse - Return use of EVT::Flag value produced by the specified 116276e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff/// SDNode. 116376e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroff/// 116476e429d781a8871afb5050f39ab8e00e72bdf2f8Steve Naroffstatic SDNode *findFlagUse(SDNode *N) { 116548a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian unsigned FlagResNo = N->getNumValues()-1; 11664afa39deaa245592977136d367251ee2c173dd8dDouglas Gregor for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 1167a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek SDUse &Use = I.getUse(); 1168a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (Use.getResNo() == FlagResNo) 11691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return Use.getUser(); 1170a135216595c00ddb5ce5f5b853a3d1f4037a13b7Fariborz Jahanian } 11711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return NULL; 1172a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek} 117317945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis 117417945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis/// findNonImmUse - Return true if "Use" is a non-immediate use of "Def". 1175653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor/// This function recursively traverses up the operand chain, ignoring 117648a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian/// certain nodes. 1177a526c5c67e5a0473c340903ee542ce570119665fTed Kremenekstatic bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, 1178a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek SDNode *Root, SmallPtrSet<SDNode*, 16> &Visited, 117948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian bool IgnoreChains) { 11806fb0aee4f9dc261bbec72e1283ad8dc0557a6d96Argyrios Kyrtzidis // The NodeID's are given uniques ID's where a node ID is guaranteed to be 1181d3a413d3b8eb39bcee5944bc545d9997c1abe492Sebastian Redl // greater than all of its (recursive) operands. If we scan to a point where 118248a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // 'use' is smaller than the node we're scanning for, then we know we will 118348a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // never find it. 1184d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // 118548a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // The Use may be -1 (unassigned) if it is a newly allocated node. This can 11861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // happen because we scan down to newly selected nodes in the case of flag 1187a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek // uses. 118817945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1)) 118917945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis return false; 1190653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor 119148a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // Don't revisit nodes if we already scanned it and didn't fail, we know we 1192a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek // won't fail if we scan it again. 1193a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (!Visited.insert(Use)) 119448a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian return false; 11956fb0aee4f9dc261bbec72e1283ad8dc0557a6d96Argyrios Kyrtzidis 11961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) { 119748a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // Ignore chain uses, they are validated by HandleMergeInputChains. 119848a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian if (Use->getOperand(i).getValueType() == MVT::Other && IgnoreChains) 1199d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer continue; 120048a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian 1201d40910b581b09c937a8c1fdcde9b8ec724398fb9Steve Naroff SDNode *N = Use->getOperand(i).getNode(); 120217945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis if (N == Def) { 12031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (Use == ImmedUse || Use == Root) 1204653f1b1bf293a9bd96fd4dd6372e779cc7af1597Douglas Gregor continue; // We are not looking for immediate use. 1205a0876e88aff05e8fc0492d216c17bff16de31a37Steve Naroff assert(N != Root); 1206d40910b581b09c937a8c1fdcde9b8ec724398fb9Steve Naroff return true; 1207d40910b581b09c937a8c1fdcde9b8ec724398fb9Steve Naroff } 1208d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 120948a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian // Traverse up the operand chain. 121048a0b6aa9d23bba2bd4ad943ad59106a91f8b3f7Fariborz Jahanian if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains)) 1211b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff return true; 1212f908a87299d278164540f90b5b6e6cab7b14fb41Steve Naroff } 1213a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek return false; 12146c6a2dbb4db96a69d6f1c2e3d322e4c3abd43814Steve Naroff} 12155086a8df9a269acc2e4f0deb1895d4a9e9d13f4dSteve Naroff 1216d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner/// IsProfitableToFold - Returns true if it's profitable to fold the specific 121732174826f58d38c20f376fe64ceac4a278133883Steve Naroff/// operand node N of U during instruction selection that starts at Root. 121832174826f58d38c20f376fe64ceac4a278133883Steve Naroffbool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U, 1219d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner SDNode *Root) const { 122032174826f58d38c20f376fe64ceac4a278133883Steve Naroff if (OptLevel == CodeGenOpt::None) return false; 122161ed9caca5dd6a9b9ccee7fb51296234e6cc68adSteve Naroff return N.hasOneUse(); 1222d9d22dd9c94618490dbffb0e2caf222530ca39d3Chris Lattner} 122332174826f58d38c20f376fe64ceac4a278133883Steve Naroff 12246c6a2dbb4db96a69d6f1c2e3d322e4c3abd43814Steve Naroff/// IsLegalToFold - Returns true if the specific operand node N of 1225a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek/// U can be folded during instruction selection that starts at Root. 12266c6a2dbb4db96a69d6f1c2e3d322e4c3abd43814Steve Naroffbool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, 1227a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek CodeGenOpt::Level OptLevel, 12281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump bool IgnoreChains) { 12291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (OptLevel == CodeGenOpt::None) return false; 123017945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis 12316327e0d55c590b3c2766fa76ef1db241a0467df2Steve Naroff // If Root use can somehow reach N through a path that that doesn't contain 12321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // U then folding N would create a cycle. e.g. In the following 123317945a0f64fe03ff6ec0c2146005a87636e3ac12Argyrios Kyrtzidis // diagram, Root can reach N through X. If N is folded into into Root, then 12346ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor // X is both a predecessor and a successor of U. 123558dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff // 12361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // [N*] // 12371eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // ^ ^ // 12386ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor // / \ // 123958dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff // [U*] [X]? // 124058dbdeb69c063f82d644504fc638120198f7fad2Steve Naroff // ^ ^ // 12412feac5e559ca5c9693a3a19905ab106b9c52e193Steve Naroff // \ / // 124273d1eb064350d5310f0475366cbe54d2d1da27bbFariborz Jahanian // \ / // 124373d1eb064350d5310f0475366cbe54d2d1da27bbFariborz Jahanian // [Root*] // 1244bef1185418705e16012b3dd50cd7c260c8d6b79cSteve Naroff // 1245bef1185418705e16012b3dd50cd7c260c8d6b79cSteve Naroff // * indicates nodes to be folded together. 1246b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // 1247b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // If Root produces a flag, then it gets (even more) interesting. Since it 1248c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // will be "glued" together with its flag use in the scheduler, we need to 1249c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // check if it might reach N. 1250c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // 1251c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // [N*] // 1252c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // ^ ^ // 1253c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // / \ // 1254c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // [U*] [X]? // 12551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // ^ ^ // 12568599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // \ \ // 12578599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // \ | // 12588599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // [Root*] | // 12598599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // ^ | // 12608599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // f | // 12618599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // | / // 126204badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // [Y] / // 126304badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // ^ / // 126404badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // f / // 126504badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // | / // 126604badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // [FU] // 126704badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // 126804badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // If FU (flag use) indirectly reaches N (the load), and Root folds N 126904badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // (call it Fold), then X is a predecessor of FU and a successor of 127004badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // Fold. But since Fold and FU are flagged together, this will create 127104badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // a cycle in the scheduling graph. 127204badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor 127304badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // If the node has flags, walk down the graph to the "lowest" node in the 127404badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor // flagged set. 127504badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor EVT VT = Root->getValueType(Root->getNumValues()-1); 127604badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor while (VT == MVT::Flag) { 127704badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor SDNode *FU = findFlagUse(Root); 127804badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor if (FU == NULL) 127904badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor break; 128004badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor Root = FU; 128104badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor VT = Root->getValueType(Root->getNumValues()-1); 1282c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff 12831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If our query node has a flag result with a use, we've walked up it. If 1284c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff // the user (which has already been selected) has a chain or indirectly uses 1285b619d957b020744bb6bfdd1cef8169d8042df43eSteve Naroff // the chain, our WalkChainUsers predicate will not consider it. Because of 1286e58ee0ca7cfad6ee75a088e7a6a090526e18da50Steve Naroff // this, we cannot ignore chains in this predicate. 12878189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek IgnoreChains = false; 12888189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } 12898189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek 1290c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff 129115f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff SmallPtrSet<SDNode*, 16> Visited; 129215f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains); 1293c77a636688e188af7e7a9a05829e542adb48e880Steve Naroff} 129415f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff 129515f081de2c8ac7deadf5d938b458b20732230cd9Steve NaroffSDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) { 129615f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff std::vector<SDValue> Ops(N->op_begin(), N->op_end()); 129715f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff SelectInlineAsmMemoryOperands(Ops); 12981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12998599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff std::vector<EVT> VTs; 13001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump VTs.push_back(MVT::Other); 13018599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff VTs.push_back(MVT::Flag); 13028599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff SDValue New = CurDAG->getNode(ISD::INLINEASM, N->getDebugLoc(), 13038599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff VTs, &Ops[0], Ops.size()); 13048599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff New->setNodeId(-1); 13058599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff return New.getNode(); 130604badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor} 130704badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor 130804badcf84c8d504d8491c7c7e29b58f52cb16640Douglas GregorSDNode *SelectionDAGISel::Select_UNDEF(SDNode *N) { 130904badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor return CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF,N->getValueType(0)); 131004badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor} 131104badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor 131204badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor/// GetVBR - decode a vbr encoding whose top bit is set. 131304badcf84c8d504d8491c7c7e29b58f52cb16640Douglas GregorALWAYS_INLINE static uint64_t 131404badcf84c8d504d8491c7c7e29b58f52cb16640Douglas GregorGetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { 131504badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor assert(Val >= 128 && "Not a VBR"); 131604badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor Val &= 127; // Remove first vbr bit. 131704badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor 131804badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor unsigned Shift = 7; 131904badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor uint64_t NextBits; 132004badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor do { 132104badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor NextBits = MatcherTable[Idx++]; 132204badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor Val |= (NextBits&127) << Shift; 132304badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor Shift += 7; 132404badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor } while (NextBits & 128); 132504badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor 132604badcf84c8d504d8491c7c7e29b58f52cb16640Douglas Gregor return Val; 132715f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff} 13284c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff 13298599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff 13308599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff/// UpdateChainsAndFlags - When a match is complete, this method updates uses of 13318599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff/// interior flag and chain results to use the new flag and chain results. 13328599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroffvoid SelectionDAGISel:: 13338599e7a394e7ed44a32dfe64733125e095e3f28cSteve NaroffUpdateChainsAndFlags(SDNode *NodeToMatch, SDValue InputChain, 13348599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff const SmallVectorImpl<SDNode*> &ChainNodesMatched, 13358599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff SDValue InputFlag, 13368599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff const SmallVectorImpl<SDNode*> &FlagResultNodesMatched, 13378599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff bool isMorphNodeTo) { 13388189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek SmallVector<SDNode*, 4> NowDeadNodes; 13398189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek 13408189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek ISelUpdater ISU(ISelPosition); 13418599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff 13428599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // Now that all the normal results are replaced, we replace the chain and 13438599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // flag results if present. 13441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (!ChainNodesMatched.empty()) { 13458189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek assert(InputChain.getNode() != 0 && 13468189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek "Matched input chains but didn't produce a chain"); 13478189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek // Loop over all of the nodes we matched that produced a chain result. 13488599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff // Replace all the chain results with the final chain we ended up with. 13498599e7a394e7ed44a32dfe64733125e095e3f28cSteve Naroff for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) { 135015f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff SDNode *ChainNode = ChainNodesMatched[i]; 135115f081de2c8ac7deadf5d938b458b20732230cd9Steve Naroff 13521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If this node was already deleted, don't look at it. 13532b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian if (ChainNode->getOpcode() == ISD::DELETED_NODE) 13542b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian continue; 1355a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 135684ed600154d48d75f204795410aae808d7e4b7bdFariborz Jahanian // Don't replace the results of the root node if we're doing a 135754055232a5ddb9529726e934301b125cb720a273Steve Naroff // MorphNodeTo. 13588f0954309eb885891908ecdfbb06d720c63128edFariborz Jahanian if (ChainNode == NodeToMatch && isMorphNodeTo) 13598189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek continue; 136084ed600154d48d75f204795410aae808d7e4b7bdFariborz Jahanian 1361ffbdead156eb0bc94d1cc7e28eadcca52dfb8194Fariborz Jahanian SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1); 1362f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff if (ChainVal.getValueType() == MVT::Flag) 1363f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2); 13641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump assert(ChainVal.getValueType() == MVT::Other && "Not a chain?"); 13656ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain, &ISU); 1366f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff 13671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If the node became dead and we haven't already seen it, delete it. 1368f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff if (ChainNode->use_empty() && 1369f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode)) 1370f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff NowDeadNodes.push_back(ChainNode); 1371d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer } 1372465d41b92b2c862f3062c412a0538db65c6a2661Abramo Bagnara } 1373df042e6c2bf06b2d9ed53c52469599ac1bd93a3fTed Kremenek 1374f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff // If the result produces a flag, update any flag results in the matched 1375f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff // pattern with the flag result. 13769d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall if (InputFlag.getNode() != 0) { 13779d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall // Handle any interior nodes explicitly marked. 13789d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall for (unsigned i = 0, e = FlagResultNodesMatched.size(); i != e; ++i) { 1379f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff SDNode *FRN = FlagResultNodesMatched[i]; 13808189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek 13818189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek // If this node was already deleted, don't look at it. 13828189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek if (FRN->getOpcode() == ISD::DELETED_NODE) 13832b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian continue; 13841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 138554055232a5ddb9529726e934301b125cb720a273Steve Naroff assert(FRN->getValueType(FRN->getNumValues()-1) == MVT::Flag && 13868189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek "Doesn't have a flag result"); 13878189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek CurDAG->ReplaceAllUsesOfValueWith(SDValue(FRN, FRN->getNumValues()-1), 13888189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek InputFlag, &ISU); 13894c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff 1390f07576130b93ba7150e04b8bc38aad96a283d074Steve Naroff // If the node became dead and we haven't already seen it, delete it. 1391c2a689b0bd0cda7bc1522dc0057f01f0897e6d3dSteve Naroff if (FRN->use_empty() && 13927e20ffe322866f6a2e820e054da9bd0f6e9b0af4Fariborz Jahanian !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), FRN)) 13933b2c58ce3b1eadc5901efff30cf8bf1885936fe7Chris Lattner NowDeadNodes.push_back(FRN); 13943b2c58ce3b1eadc5901efff30cf8bf1885936fe7Chris Lattner } 13953b2c58ce3b1eadc5901efff30cf8bf1885936fe7Chris Lattner } 13961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13973b2c58ce3b1eadc5901efff30cf8bf1885936fe7Chris Lattner if (!NowDeadNodes.empty()) 1398c2a689b0bd0cda7bc1522dc0057f01f0897e6d3dSteve Naroff CurDAG->RemoveDeadNodes(NowDeadNodes, &ISU); 139984472a897a99db80d49eb5ac1ac85d54b1cc6554Steve Naroff 14009f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff DEBUG(errs() << "ISEL: Match complete!\n"); 14011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 14029f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff 14039f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroffenum ChainResult { 140426337b2f6d4b8dc7458a6e24dca05cb021a21861Fariborz Jahanian CR_Simple, 1405c374cd978abdeb289d360426129886f6e354a459Fariborz Jahanian CR_InducesCycle, 1406c374cd978abdeb289d360426129886f6e354a459Fariborz Jahanian CR_LeadsToInteriorNode 14079f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff}; 14089f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff 14091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// WalkChainUsers - Walk down the users of the specified chained node that is 14106ab3524f72a6e64aa04973fa9433b5559abb3525Douglas Gregor/// part of the pattern we're matching, looking at all of the users we find. 14119f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff/// This determines whether something is an interior node, whether we have a 14121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// non-pattern node in between two pattern nodes (which prevent folding because 14139f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff/// it would induce a cycle) and whether we have a TokenFactor node sandwiched 14149f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff/// between pattern nodes (in which case the TF becomes part of the pattern). 14159f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff/// 1416d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer/// The walk we do here is guaranteed to be small because we quickly get down to 1417465d41b92b2c862f3062c412a0538db65c6a2661Abramo Bagnara/// already selected nodes "below" us. 1418df042e6c2bf06b2d9ed53c52469599ac1bd93a3fTed Kremenekstatic ChainResult 14199f525970bec0e29f32cec6141aa3baa245c5ae82Steve NaroffWalkChainUsers(SDNode *ChainedNode, 14209f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff SmallVectorImpl<SDNode*> &ChainedNodesInPattern, 14219d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall SmallVectorImpl<SDNode*> &InteriorChainedNodes) { 14229d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall ChainResult Result = CR_Simple; 14239d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall 14249f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff for (SDNode::use_iterator UI = ChainedNode->use_begin(), 14258189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek E = ChainedNode->use_end(); UI != E; ++UI) { 14268d36616a093f65f667d22bc1136a4c2be0bae7dfChris Lattner // Make sure the use is of the chain, not some other value we produce. 14272b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian if (UI.getUse().getValueType() != MVT::Other) continue; 14289f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff 14299f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff SDNode *User = *UI; 14309f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff 14311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If we see an already-selected machine node, then we've gone beyond the 14329f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff // pattern that we're selecting down into the already selected chunk of the 14339f525970bec0e29f32cec6141aa3baa245c5ae82Steve Naroff // DAG. 1434c2a689b0bd0cda7bc1522dc0057f01f0897e6d3dSteve Naroff if (User->isMachineOpcode() || 143584472a897a99db80d49eb5ac1ac85d54b1cc6554Steve Naroff User->getOpcode() == ISD::HANDLENODE) // Root of the graph. 14367e3411b23eb3724da5461dc6d6f97d14b3b0a52fSteve Naroff continue; 14377e3411b23eb3724da5461dc6d6f97d14b3b0a52fSteve Naroff 14382b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian if (User->getOpcode() == ISD::CopyToReg || 14392b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian User->getOpcode() == ISD::CopyFromReg || 14402b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian User->getOpcode() == ISD::INLINEASM || 14412b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian User->getOpcode() == ISD::EH_LABEL) { 14422b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // If their node ID got reset to -1 then they've already been selected. 14432b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // Treat them like a MachineOpcode. 14442b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian if (User->getNodeId() == -1) 14452b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian continue; 14462b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian } 14472b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian 14482b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // If we have a TokenFactor, we handle it specially. 14492b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian if (User->getOpcode() != ISD::TokenFactor) { 14502b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // If the node isn't a token factor and isn't part of our pattern, then it 14512b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // must be a random chained node in between two nodes we're selecting. 1452376338a68d467f076c2ae5fa614ec0286ea74d7cFariborz Jahanian // This happens when we have something like: 1453376338a68d467f076c2ae5fa614ec0286ea74d7cFariborz Jahanian // x = load ptr 1454376338a68d467f076c2ae5fa614ec0286ea74d7cFariborz Jahanian // call 1455376338a68d467f076c2ae5fa614ec0286ea74d7cFariborz Jahanian // y = x+4 1456376338a68d467f076c2ae5fa614ec0286ea74d7cFariborz Jahanian // store y -> ptr 14572b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // Because we structurally match the load/store as a read/modify/write, 14582b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // but the call is chained between them. We cannot fold in this case 14592b9b0b2c9a7137f46b7cc6a02ca9608f2fcc868dFariborz Jahanian // because it would induce a cycle in the graph. 1460fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff if (!std::count(ChainedNodesInPattern.begin(), 1461fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff ChainedNodesInPattern.end(), User)) 1462fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff return CR_InducesCycle; 14631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1464fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // Otherwise we found a node that is part of our pattern. For example in: 14651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // x = load ptr 14661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // y = x+4 1467fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // store y -> ptr 1468fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // This would happen when we're scanning down from the load and see the 1469fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // store as a user. Record that there is a use of ChainedNode that is 1470fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // part of the pattern and keep scanning uses. 1471fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff Result = CR_LeadsToInteriorNode; 1472fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff InteriorChainedNodes.push_back(User); 1473fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff continue; 1474fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff } 1475fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff 1476fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // If we found a TokenFactor, there are two cases to consider: first if the 1477fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // TokenFactor is just hanging "below" the pattern we're matching (i.e. no 1478fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // uses of the TF are in our pattern) we just want to ignore it. Second, 1479b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff // the TokenFactor can be sandwiched in between two chained nodes, like so: 1480fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // [Load chain] 1481fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // ^ 1482fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // | 1483fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // [Load] 1484fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // ^ ^ 1485fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // | \ DAG's like cheese 1486fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // / \ do you? 1487fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // / | 14881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // [TokenFactor] [Op] 1489fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // ^ ^ 1490fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // | | 1491fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // \ / 1492d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // \ / 14932bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff // [Store] 1494fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // 1495fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // In this case, the TokenFactor becomes part of our match and we rewrite it 1496a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian // as a new TokenFactor. 1497fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // 1498fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // To distinguish these two cases, do a recursive walk down the uses. 1499fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff switch (WalkChainUsers(User, ChainedNodesInPattern, InteriorChainedNodes)) { 1500fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff case CR_Simple: 1501fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // If the uses of the TokenFactor are just already-selected nodes, ignore 1502fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // it, it is "below" our pattern. 1503fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff continue; 1504fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff case CR_InducesCycle: 15051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If the uses of the TokenFactor lead to nodes that are not part of our 1506fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // pattern that are not selected, folding would turn this into a cycle, 1507fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff // bail out now. 1508fa15fd9db405f2d0a9811e94a671f7d7bc282385Steve Naroff return CR_InducesCycle; 1509d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer case CR_LeadsToInteriorNode: 15101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump break; // Otherwise, keep processing. 1511e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian } 1512e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian 1513e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian // Okay, we know we're in the interesting interior case. The TokenFactor 1514a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian // is now going to be considered part of the pattern so that we rewrite its 151510d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian // uses (it may have uses that are not part of the pattern) with the 151610d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian // ultimate chain result of the generated code. We will also add its chain 15171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // inputs as inputs to the ultimate TokenFactor we create. 151810d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian Result = CR_LeadsToInteriorNode; 151910d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian ChainedNodesInPattern.push_back(User); 15201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InteriorChainedNodes.push_back(User); 152133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian continue; 152210d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian } 152333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 15241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return Result; 152533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian} 152610d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian 152710d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian/// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains 152810d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian/// operation for when the pattern matched at least one node with a chains. The 152910d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian/// input vector contains a list of all of the chained nodes that we match. We 153010d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian/// must determine if this is a valid thing to cover (i.e. matching it won't 15311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// induce cycles in the DAG) and if so, creating a TokenFactor node. that will 153233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian/// be used as the input node chain for the generated nodes. 153388f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanianstatic SDValue 153410d24f062ee8c83823199191ccdba452c7f9ac24Fariborz JahanianHandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched, 1535e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian SelectionDAG *CurDAG) { 153610d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian // Walk all of the chained nodes we've matched, recursively scanning down the 15371eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // users of the chain result. This adds any TokenFactor nodes that are caught 153833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // in between chained nodes to the chained and interior nodes list. 153933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SmallVector<SDNode*, 3> InteriorChainedNodes; 1540e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) { 154110d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian if (WalkChainUsers(ChainNodesMatched[i], ChainNodesMatched, 154210d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian InteriorChainedNodes) == CR_InducesCycle) 154310d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian return SDValue(); // Would induce a cycle. 154410d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian } 154510d24f062ee8c83823199191ccdba452c7f9ac24Fariborz Jahanian 1546b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff // Okay, we have walked all the matched nodes and collected TokenFactor nodes 1547338d1e2ced8037b71d91fb319631846917d0cedaChris Lattner // that we are interested in. Form our input TokenFactor node. 1548e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian SmallVector<SDValue, 3> InputChains; 15491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) { 1550e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian // Add the input chain of this node to the InputChains list (which will be 15511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // the operands of the generated TokenFactor) if it's not an interior node. 1552e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian SDNode *N = ChainNodesMatched[i]; 15531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (N->getOpcode() != ISD::TokenFactor) { 155433b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N)) 155533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian continue; 155633b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 155788f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian // Otherwise, add the input chain. 155833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SDValue InChain = ChainNodesMatched[i]->getOperand(0); 155933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian assert(InChain.getValueType() == MVT::Other && "Not a chain"); 156033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian InputChains.push_back(InChain); 156133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian continue; 15627e24e82a70a2c681f4291a3397bcd1e1005f251aChris Lattner } 15631ed8e2a39456d064a346a6a5c2f6bd53e6d57345Ted Kremenek 1564e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff // If we have a token factor, we want to add all inputs of the token factor 1565e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff // that are not part of the pattern we're matching. 1566e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1567e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(), 1568e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff N->getOperand(op).getNode())) 1569fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar InputChains.push_back(N->getOperand(op)); 157088f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian } 157133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian } 15728ec03f58c33c33a917f54bb7f2cd61b6d7ffe0caChris Lattner 157333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SDValue Res; 157433b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if (InputChains.size() == 1) 157533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return InputChains[0]; 157606767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner return CurDAG->getNode(ISD::TokenFactor, ChainNodesMatched[0]->getDebugLoc(), 157706767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner MVT::Other, &InputChains[0], InputChains.size()); 15788ec03f58c33c33a917f54bb7f2cd61b6d7ffe0caChris Lattner} 1579e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff 1580e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff/// MorphNode - Handle morphing a node in place for the selector. 1581e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve NaroffSDNode *SelectionDAGISel:: 1582e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve NaroffMorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, 1583e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff const SDValue *Ops, unsigned NumOps, unsigned EmitNodeInfo) { 1584e89b8e7946155db1a3bbb1e22addd5d9d2a904dfSteve Naroff // It is possible we're using MorphNodeTo to replace a node with no 1585fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar // normal results with one that has a normal result (or we could be 158688f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian // adding a chain) and the input could have flags and chains as well. 15871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // In this case we need to shift the operands down. 158833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // FIXME: This is a horrible hack and broken in obscure cases, no worse 158933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // than the old isel though. 159033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian int OldFlagResultNo = -1, OldChainResultNo = -1; 159133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 159233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian unsigned NTMNumResults = Node->getNumValues(); 159333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if (Node->getValueType(NTMNumResults-1) == MVT::Flag) { 15947571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian OldFlagResultNo = NTMNumResults-1; 15957571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian if (NTMNumResults != 1 && 15967571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian Node->getValueType(NTMNumResults-2) == MVT::Other) 15977571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian OldChainResultNo = NTMNumResults-2; 15987571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian } else if (Node->getValueType(NTMNumResults-1) == MVT::Other) 15997571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian OldChainResultNo = NTMNumResults-1; 16007571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian 16017571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian // Call the underlying SelectionDAG routine to do the transmogrification. Note 16027571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian // that this deletes operands of the old node that become dead. 16037571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops, NumOps); 16047571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian 16057571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian // MorphNodeTo can operate in two ways: if an existing node with the 16061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // specified operands exists, it can just return it. Otherwise, it 16071eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // updates the node in place to have the requested operands. 1608d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (Res == Node) { 160933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // If we updated the node in place, reset the node ID. To the isel, 16107571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian // this should be just like a newly allocated machine node. 16117571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian Res->setNodeId(-1); 16127571228ec9b31de7733ae96352ee4eaa8afcf2b5Fariborz Jahanian } 161333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 16141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump unsigned ResNumResults = Res->getNumValues(); 161533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // Move the flag if needed. 161633b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if ((EmitNodeInfo & OPFL_FlagOutput) && OldFlagResultNo != -1 && 161733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian (unsigned)OldFlagResultNo != ResNumResults-1) 16181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldFlagResultNo), 161933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SDValue(Res, ResNumResults-1)); 162033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 16211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if ((EmitNodeInfo & OPFL_FlagOutput) != 0) 162233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian --ResNumResults; 16231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // Move the chain reference if needed. 162533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 && 162633b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian (unsigned)OldChainResultNo != ResNumResults-1) 162733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo), 162833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SDValue(Res, ResNumResults-1)); 162933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 163033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // Otherwise, no replacement happened because the node already exists. Replace 163133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // Uses of the old node with the new one. 163233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if (Res != Node) 163333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian CurDAG->ReplaceAllUsesWith(Node, Res); 16341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 163533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return Res; 163688f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian} 163733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 163833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian/// CheckPatternPredicate - Implements OP_CheckPatternPredicate. 163933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianALWAYS_INLINE static bool 164033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianCheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, 164133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SDValue N, const SmallVectorImpl<SDValue> &RecordedNodes) { 164233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian // Accept if it is exactly the same as a previously recorded node. 164333b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian unsigned RecNo = MatcherTable[MatcherIndex++]; 164433b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 164588f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian return N == RecordedNodes[RecNo]; 164688f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian} 164788f50f31d84185827a8f765dd611134bcaface7bFariborz Jahanian 164833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian/// CheckPatternPredicate - Implements OP_CheckPatternPredicate. 1649d999b3736ce2646ae0711416b421d906298764dbBenjamin KramerALWAYS_INLINE static bool 16501eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpCheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1651e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian SelectionDAGISel &SDISel) { 165233b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]); 16531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 165433b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 165533b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian/// CheckNodePredicate - Implements OP_CheckNodePredicate. 1656e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz JahanianALWAYS_INLINE static bool 165733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianCheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, 165833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian SelectionDAGISel &SDISel, SDNode *N) { 165933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]); 166033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian} 16611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1662e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz JahanianALWAYS_INLINE static bool 1663e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz JahanianCheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1664e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian SDNode *N) { 1665e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian uint16_t Opc = MatcherTable[MatcherIndex++]; 1666e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; 166733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return N->getOpcode() == Opc; 166833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian} 166933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 167033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianALWAYS_INLINE static bool 167133b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianCheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, 167265b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz Jahanian SDValue N, const TargetLowering &TLI) { 167365b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz Jahanian MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 167465b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz Jahanian if (N.getValueType() == VT) return true; 1675e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian 1676e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian // Handle the case when VT is iPTR. 1677e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian return VT == MVT::iPTR && N.getValueType() == TLI.getPointerTy(); 167833b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian} 167933b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 168033b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz JahanianALWAYS_INLINE static bool 168165b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz JahanianCheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, 168265b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz Jahanian SDValue N, const TargetLowering &TLI, 168365b0aa5d3973e07c8c4449ef2b81c389fd48229aFariborz Jahanian unsigned ChildNo) { 168433b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian if (ChildNo >= N.getNumOperands()) 16851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return false; // Match fails if out of range child #. 168633b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI); 1687d3a413d3b8eb39bcee5944bc545d9997c1abe492Sebastian Redl} 1688600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff 1689600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff 1690d999b3736ce2646ae0711416b421d906298764dbBenjamin KramerALWAYS_INLINE static bool 1691600e4e8c358fd39e70f3313b7f1bfa0848957597Steve NaroffCheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1692600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff SDValue N) { 1693600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff return cast<CondCodeSDNode>(N)->get() == 1694600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff (ISD::CondCode)MatcherTable[MatcherIndex++]; 1695600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff} 1696600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff 1697600e4e8c358fd39e70f3313b7f1bfa0848957597Steve NaroffALWAYS_INLINE static bool 1698600e4e8c358fd39e70f3313b7f1bfa0848957597Steve NaroffCheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1699600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff SDValue N, const TargetLowering &TLI) { 1700600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 1701600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff if (cast<VTSDNode>(N)->getVT() == VT) 1702600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff return true; 1703d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 1704600e4e8c358fd39e70f3313b7f1bfa0848957597Steve Naroff // Handle the case when VT is iPTR. 1705e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI.getPointerTy(); 1706e8d1c0579404442a87818506bb0e742d0f52d5bfFariborz Jahanian} 170733b9c4e4c6f1b055d94c8b57cd16883125701ac7Fariborz Jahanian 170810d24f062ee8c83823199191ccdba452c7f9ac24Fariborz JahanianALWAYS_INLINE static bool 170910d24f062ee8c83823199191ccdba452c7f9ac24Fariborz JahanianCheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, 17101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SDValue N) { 1711a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian int64_t Val = MatcherTable[MatcherIndex++]; 1712a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian if (Val & 128) 1713a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian Val = GetVBR(Val, MatcherTable, MatcherIndex); 1714a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian 1715a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian ConstantSDNode *C = dyn_cast<ConstantSDNode>(N); 1716b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff return C != 0 && C->getSExtValue() == Val; 1717a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian} 1718a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian 1719a0f55792409289d1d343023fa8292cff6355e538Fariborz JahanianALWAYS_INLINE static bool 17201eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpCheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1721a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian SDValue N, SelectionDAGISel &SDISel) { 17221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump int64_t Val = MatcherTable[MatcherIndex++]; 17231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (Val & 128) 17243498cc9793487a5ad51a5a050be85bd550775eccSteve Naroff Val = GetVBR(Val, MatcherTable, MatcherIndex); 17253498cc9793487a5ad51a5a050be85bd550775eccSteve Naroff 17263498cc9793487a5ad51a5a050be85bd550775eccSteve Naroff if (N->getOpcode() != ISD::AND) return false; 1727d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 17281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 17291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return C != 0 && SDISel.CheckAndMask(N.getOperand(0), C, Val); 1730c7089f1be946f3ca6e59596f0e0f92d96136e4c8Steve Naroff} 1731c7089f1be946f3ca6e59596f0e0f92d96136e4c8Steve Naroff 1732a0f55792409289d1d343023fa8292cff6355e538Fariborz JahanianALWAYS_INLINE static bool 1733c7089f1be946f3ca6e59596f0e0f92d96136e4c8Steve NaroffCheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, 1734c7089f1be946f3ca6e59596f0e0f92d96136e4c8Steve Naroff SDValue N, SelectionDAGISel &SDISel) { 1735a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian int64_t Val = MatcherTable[MatcherIndex++]; 1736a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian if (Val & 128) 1737a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian Val = GetVBR(Val, MatcherTable, MatcherIndex); 1738a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian 1739a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian if (N->getOpcode() != ISD::OR) return false; 1740a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian 1741a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1742a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian return C != 0 && SDISel.CheckOrMask(N.getOperand(0), C, Val); 1743d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer} 1744a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian 1745a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian/// IsPredicateKnownToFail - If we know how and can do so without pushing a 17461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// scope, evaluate the current node. If the current predicate is known to 1747c7089f1be946f3ca6e59596f0e0f92d96136e4c8Steve Naroff/// fail, set Result=true and return anything. If the current predicate is 1748a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian/// known to pass, set Result=false and return the MatcherIndex to continue 1749a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian/// with. If the current predicate is unknown, set Result=false and return the 1750a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian/// MatcherIndex to continue with. 1751621edce9cd7d4e06979daf911cc306350619f33bSteve Naroffstatic unsigned IsPredicateKnownToFail(const unsigned char *Table, 1752621edce9cd7d4e06979daf911cc306350619f33bSteve Naroff unsigned Index, SDValue N, 1753a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian bool &Result, SelectionDAGISel &SDISel, 1754b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff SmallVectorImpl<SDValue> &RecordedNodes){ 1755b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff switch (Table[Index++]) { 1756b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff default: 17579d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall Result = false; 17589d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall return Index-1; // Could not evaluate this predicate. 17599d125033a9853f3b572a4c9e2f9e2d4e5e346973John McCall case SelectionDAGISel::OPC_CheckSame: 1760a95d3750441ac8ad03e36af8e6e74039c9a3109dTed Kremenek Result = !::CheckSame(Table, Index, N, RecordedNodes); 1761a95d3750441ac8ad03e36af8e6e74039c9a3109dTed Kremenek return Index; 1762e4f2142d00fa5fdb580c4e2413da91882d955381Chris Lattner case SelectionDAGISel::OPC_CheckPatternPredicate: 1763e4f2142d00fa5fdb580c4e2413da91882d955381Chris Lattner Result = !::CheckPatternPredicate(Table, Index, SDISel); 1764b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return Index; 1765b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckPredicate: 1766b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode()); 1767b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return Index; 1768b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckOpcode: 1769a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian Result = !::CheckOpcode(Table, Index, N.getNode()); 1770a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian return Index; 17711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case SelectionDAGISel::OPC_CheckType: 1772d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer Result = !::CheckType(Table, Index, N, SDISel.TLI); 1773b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return Index; 1774b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckChild0Type: 1775b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckChild1Type: 1776b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckChild2Type: 1777b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckChild3Type: 1778b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckChild4Type: 1779a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian case SelectionDAGISel::OPC_CheckChild5Type: 1780a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian case SelectionDAGISel::OPC_CheckChild6Type: 1781a0f55792409289d1d343023fa8292cff6355e538Fariborz Jahanian case SelectionDAGISel::OPC_CheckChild7Type: 1782b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff Result = !::CheckChildType(Table, Index, N, SDISel.TLI, 1783b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Type); 17848c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff return Index; 17858c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff case SelectionDAGISel::OPC_CheckCondCode: 17868c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff Result = !::CheckCondCode(Table, Index, N); 17878c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff return Index; 1788b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckValueType: 17898c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff Result = !::CheckValueType(Table, Index, N, SDISel.TLI); 1790b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return Index; 17911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case SelectionDAGISel::OPC_CheckInteger: 17928c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff Result = !::CheckInteger(Table, Index, N); 17938c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff return Index; 17948c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff case SelectionDAGISel::OPC_CheckAndImm: 17958c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff Result = !::CheckAndImm(Table, Index, N, SDISel); 17968c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff return Index; 1797b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case SelectionDAGISel::OPC_CheckOrImm: 1798b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff Result = !::CheckOrImm(Table, Index, N, SDISel); 1799b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff return Index; 1800b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff } 1801b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff} 1802b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1803b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroffnamespace { 1804b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1805b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroffstruct MatchScope { 1806b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// FailIndex - If this match fails, this is the index to continue with. 1807b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff unsigned FailIndex; 1808b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1809b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// NodeStack - The node stack when the scope was formed. 1810b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff SmallVector<SDValue, 4> NodeStack; 1811b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1812b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// NumRecordedNodes - The number of recorded nodes when the scope was formed. 1813b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff unsigned NumRecordedNodes; 1814b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1815b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// NumMatchedMemRefs - The number of matched memref entries. 1816b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff unsigned NumMatchedMemRefs; 1817b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1818b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// InputChain/InputFlag - The current chain/flag 1819b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff SDValue InputChain, InputFlag; 1820b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1821b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty. 1822b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff bool HasChainNodesMatched, HasFlagResultNodesMatched; 1823b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff}; 1824b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1825b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff} 1826b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 1827b85e77af052ea495aa47edcce3b1ec8887e53873Steve NaroffSDNode *SelectionDAGISel:: 1828d999b3736ce2646ae0711416b421d906298764dbBenjamin KramerSelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, 1829d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer unsigned TableSize) { 1830b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff // FIXME: Should these even be selected? Handle these cases in the caller? 1831b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff switch (NodeToMatch->getOpcode()) { 1832b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff default: 1833b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff break; 1834b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::EntryToken: // These nodes remain the same. 1835b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::BasicBlock: 1836b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::Register: 1837b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff //case ISD::VALUETYPE: 1838b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff //case ISD::CONDCODE: 1839b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::HANDLENODE: 1840b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::MDNODE_SDNODE: 1841b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetConstant: 1842b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetConstantFP: 1843b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetConstantPool: 1844b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetFrameIndex: 1845b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetExternalSymbol: 1846b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetBlockAddress: 1847b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetJumpTable: 1848b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetGlobalTLSAddress: 1849b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TargetGlobalAddress: 1850b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::TokenFactor: 1851b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::CopyFromReg: 1852b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::CopyToReg: 1853b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::EH_LABEL: 1854d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer NodeToMatch->setNodeId(-1); // Mark selected. 1855d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer return 0; 1856b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::AssertSext: 1857b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case ISD::AssertZext: 1858b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, 0), 1859b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff NodeToMatch->getOperand(0)); 1860b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff return 0; 18617573098b83e780d1c5bea13b384b610d8f155676Steve Naroff case ISD::INLINEASM: return Select_INLINEASM(NodeToMatch); 18627573098b83e780d1c5bea13b384b610d8f155676Steve Naroff case ISD::UNDEF: return Select_UNDEF(NodeToMatch); 18637573098b83e780d1c5bea13b384b610d8f155676Steve Naroff } 18641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18657573098b83e780d1c5bea13b384b610d8f155676Steve Naroff assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); 18667573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 18677573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Set up the node stack with NodeToMatch as the only node on the stack. 18687573098b83e780d1c5bea13b384b610d8f155676Steve Naroff SmallVector<SDValue, 8> NodeStack; 18697573098b83e780d1c5bea13b384b610d8f155676Steve Naroff SDValue N = SDValue(NodeToMatch, 0); 18707573098b83e780d1c5bea13b384b610d8f155676Steve Naroff NodeStack.push_back(N); 18717573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 18727573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // MatchScopes - Scopes used when matching, if a match failure happens, this 18737573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // indicates where to continue checking. 187421867b1188ae19b2834c155b21e97c39ed1b9770Steve Naroff SmallVector<MatchScope, 8> MatchScopes; 18757573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 1876d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // RecordedNodes - This is the set of nodes that have been recorded by the 18771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // state machine. 18787573098b83e780d1c5bea13b384b610d8f155676Steve Naroff SmallVector<SDValue, 8> RecordedNodes; 18797573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 18807573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // MatchedMemRefs - This is the set of MemRef's we've seen in the input 18817573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // pattern. 18821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SmallVector<MachineMemOperand*, 2> MatchedMemRefs; 1883c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff 18848f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor // These are the current input chain and flag for use when generating nodes. 1885c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff // Various Emit operations change these. For example, emitting a copytoreg 1886c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff // uses and updates these. 1887c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff SDValue InputChain, InputFlag; 1888c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff 1889c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff // ChainNodesMatched - If a pattern matches nodes that have input/output 1890c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff // chains, the OPC_EmitMergeInputChains operation is emitted which indicates 1891c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff // which ones they are. The result is captured into this list so that we can 18921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // update the chain results when the pattern is complete. 1893d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer SmallVector<SDNode*, 3> ChainNodesMatched; 18948bd3dc64d28bdb3b263838a12aca25587bce661bSteve Naroff SmallVector<SDNode*, 3> FlagResultNodesMatched; 18958bd3dc64d28bdb3b263838a12aca25587bce661bSteve Naroff 18968bd3dc64d28bdb3b263838a12aca25587bce661bSteve Naroff DEBUG(errs() << "ISEL: Starting pattern match on root node: "; 18978bd3dc64d28bdb3b263838a12aca25587bce661bSteve Naroff NodeToMatch->dump(CurDAG); 1898d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer errs() << '\n'); 1899c9ba172447f2e2fe938ae03c3b16b91bda5b4dc3Steve Naroff 19007573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Determine where to start the interpreter. Normally we start at opcode #0, 19017573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // but if the state machine starts with an OPC_SwitchOpcode, then we 19028f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor // accelerate the first lookup (which is guaranteed to be hot) with the 19038f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor // OpcodeOffset table. 1904c00d8e18ad3d903acfeb5d05163ce90713066a3fDouglas Gregor unsigned MatcherIndex = 0; 19057573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 19068f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor if (!OpcodeOffset.empty()) { 19077573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Already computed the OpcodeOffset table, just index into it. 19087573098b83e780d1c5bea13b384b610d8f155676Steve Naroff if (N.getOpcode() < OpcodeOffset.size()) 19097573098b83e780d1c5bea13b384b610d8f155676Steve Naroff MatcherIndex = OpcodeOffset[N.getOpcode()]; 19108f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor DEBUG(errs() << " Initial Opcode index to " << MatcherIndex << "\n"); 19117573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 19121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else if (MatcherTable[0] == OPC_SwitchOpcode) { 19137573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Otherwise, the table isn't computed, but the state machine does start 19141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // with an OPC_SwitchOpcode instruction. Populate the table now, since this 19157573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // is the first time we're selecting an instruction. 19167573098b83e780d1c5bea13b384b610d8f155676Steve Naroff unsigned Idx = 1; 19178f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor while (1) { 1918e12e69223ec279e89ac681023d42184b115e0068Steve Naroff // Get the size of this case. 19198f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor unsigned CaseSize = MatcherTable[Idx++]; 1920e12e69223ec279e89ac681023d42184b115e0068Steve Naroff if (CaseSize & 128) 1921e12e69223ec279e89ac681023d42184b115e0068Steve Naroff CaseSize = GetVBR(CaseSize, MatcherTable, Idx); 19228f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor if (CaseSize == 0) break; 192306767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner 1924e12e69223ec279e89ac681023d42184b115e0068Steve Naroff // Get the opcode, add the index to the table. 19251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump uint16_t Opc = MatcherTable[Idx++]; 1926e12e69223ec279e89ac681023d42184b115e0068Steve Naroff Opc |= (unsigned short)MatcherTable[Idx++] << 8; 1927d7407dc92c7d19cafce429e7e1cf9819d3fc0b92Daniel Dunbar if (Opc >= OpcodeOffset.size()) 19287ba138abd329e591a8f6d5001f60dd7082f71b3bSteve Naroff OpcodeOffset.resize((Opc+1)*2); 19297ba138abd329e591a8f6d5001f60dd7082f71b3bSteve Naroff OpcodeOffset[Opc] = Idx; 1930a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek Idx += CaseSize; 19317573098b83e780d1c5bea13b384b610d8f155676Steve Naroff } 1932d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 19337573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Okay, do the lookup for the first opcode. 1934506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall if (N.getOpcode() < OpcodeOffset.size()) 1935506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall MatcherIndex = OpcodeOffset[N.getOpcode()]; 1936506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall } 1937506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall 1938506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall while (1) { 193921867b1188ae19b2834c155b21e97c39ed1b9770Steve Naroff assert(MatcherIndex < TableSize && "Invalid index"); 1940506b57e8b79d7dc2c367bf2ee7ec95420ad3fc8fJohn McCall#ifndef NDEBUG 194121867b1188ae19b2834c155b21e97c39ed1b9770Steve Naroff unsigned CurrentOpcodeIndex = MatcherIndex; 1942d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer#endif 19437573098b83e780d1c5bea13b384b610d8f155676Steve Naroff BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++]; 19447573098b83e780d1c5bea13b384b610d8f155676Steve Naroff switch (Opcode) { 19457573098b83e780d1c5bea13b384b610d8f155676Steve Naroff case OPC_Scope: { 19468f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor // Okay, the semantics of this operation are that we should push a scope 19478f5e3dd32e443768d9dbbad7191e123e6733750cDouglas Gregor // then evaluate the first child. However, pushing a scope only to have 19487573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // the first check fail (which then pops it) is inefficient. If we can 19497573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // determine immediately that the first check (or first several) will 19507573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // immediately fail, don't even bother pushing a scope for them. 19517573098b83e780d1c5bea13b384b610d8f155676Steve Naroff unsigned FailIndex; 19527573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 19531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump while (1) { 19541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump unsigned NumToSkip = MatcherTable[MatcherIndex++]; 19557573098b83e780d1c5bea13b384b610d8f155676Steve Naroff if (NumToSkip & 128) 1956d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex); 19577ba138abd329e591a8f6d5001f60dd7082f71b3bSteve Naroff // Found the end of the scope with no match. 19587573098b83e780d1c5bea13b384b610d8f155676Steve Naroff if (NumToSkip == 0) { 19592bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff FailIndex = 0; 19607573098b83e780d1c5bea13b384b610d8f155676Steve Naroff break; 19617573098b83e780d1c5bea13b384b610d8f155676Steve Naroff } 19627573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 19637573098b83e780d1c5bea13b384b610d8f155676Steve Naroff FailIndex = MatcherIndex+NumToSkip; 196406767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner 196506767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner unsigned MatcherIndexOfPredicate = MatcherIndex; 19661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump (void)MatcherIndexOfPredicate; // silence warning. 1967378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff 1968378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff // If we can't evaluate this predicate without pushing a scope (e.g. if 1969378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff // it is a 'MoveParent') or if the predicate succeeds on this node, we 1970378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff // push the scope and evaluate the full predicate chain. 1971378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff bool Result; 1972378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N, 1973378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff Result, *this, RecordedNodes); 1974378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff if (!Result) 1975378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff break; 1976d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 19771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump DEBUG(errs() << " Skipped scope entry (due to false predicate) at " 19787573098b83e780d1c5bea13b384b610d8f155676Steve Naroff << "index " << MatcherIndexOfPredicate 19797573098b83e780d1c5bea13b384b610d8f155676Steve Naroff << ", continuing at " << FailIndex << "\n"); 19807573098b83e780d1c5bea13b384b610d8f155676Steve Naroff ++NumDAGIselRetries; 1981a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 19827573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Otherwise, we know that this case of the Scope is guaranteed to fail, 19837573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // move to the next case. 19847573098b83e780d1c5bea13b384b610d8f155676Steve Naroff MatcherIndex = FailIndex; 19851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 1986d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 19871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // If the whole scope failed to match, bail. 19887573098b83e780d1c5bea13b384b610d8f155676Steve Naroff if (FailIndex == 0) break; 19897573098b83e780d1c5bea13b384b610d8f155676Steve Naroff 19907573098b83e780d1c5bea13b384b610d8f155676Steve Naroff // Push a MatchScope which indicates where to go if the first child fails 199106767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner // to match. 199206767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner MatchScope NewEntry; 19931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NewEntry.FailIndex = FailIndex; 199406767518f56cc7ed2a1ae4a37434ad9bdd7890beChris Lattner NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end()); 19951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NewEntry.NumRecordedNodes = RecordedNodes.size(); 19967573098b83e780d1c5bea13b384b610d8f155676Steve Naroff NewEntry.NumMatchedMemRefs = MatchedMemRefs.size(); 1997d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer NewEntry.InputChain = InputChain; 19987573098b83e780d1c5bea13b384b610d8f155676Steve Naroff NewEntry.InputFlag = InputFlag; 1999d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty(); 20001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NewEntry.HasFlagResultNodesMatched = !FlagResultNodesMatched.empty(); 20017573098b83e780d1c5bea13b384b610d8f155676Steve Naroff MatchScopes.push_back(NewEntry); 20027573098b83e780d1c5bea13b384b610d8f155676Steve Naroff continue; 20031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 20048c56515a0c61b73fc2f02cc96dc3e37650d89d45Steve Naroff case OPC_RecordNode: 2005b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff // Remember this node, it may end up being an operand in the pattern. 2006378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff RecordedNodes.push_back(N); 2007378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff continue; 2008378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff 2009378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff case OPC_RecordChild0: case OPC_RecordChild1: 2010378f47afeebe3fde2c17e78ae1cf42d2e3beeacfSteve Naroff case OPC_RecordChild2: case OPC_RecordChild3: 2011d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer case OPC_RecordChild4: case OPC_RecordChild5: 2012b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff case OPC_RecordChild6: case OPC_RecordChild7: { 2013b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff unsigned ChildNo = Opcode-OPC_RecordChild0; 2014b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff if (ChildNo >= N.getNumOperands()) 2015b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff break; // Match fails if out of range child #. 2016b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff 2017b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff RecordedNodes.push_back(N->getOperand(ChildNo)); 2018b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff continue; 2019b85e77af052ea495aa47edcce3b1ec8887e53873Steve Naroff } 20207573098b83e780d1c5bea13b384b610d8f155676Steve Naroff case OPC_RecordMemRef: 20217573098b83e780d1c5bea13b384b610d8f155676Steve Naroff MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand()); 20227573098b83e780d1c5bea13b384b610d8f155676Steve Naroff continue; 2023d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer 2024909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian case OPC_CaptureFlagInput: 2025909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian // If the current node has an input flag, capture it in InputFlag. 2026909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian if (N->getNumOperands() != 0 && 2027b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) 2028909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian InputFlag = N->getOperand(N->getNumOperands()-1); 2029909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian continue; 2030909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian 2031b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff case OPC_MoveChild: { 2032909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian unsigned ChildNo = MatcherTable[MatcherIndex++]; 2033909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian if (ChildNo >= N.getNumOperands()) 2034909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian break; // Match fails if out of range child #. 20351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump N = N.getOperand(ChildNo); 20361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NodeStack.push_back(N); 20372bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff continue; 2038b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff } 20392bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff 20402bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff case OPC_MoveParent: 20412bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff // Pop the current node off the NodeStack. 20421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NodeStack.pop_back(); 20432bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff assert(!NodeStack.empty() && "Node stack imbalance!"); 20442bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff N = NodeStack.back(); 20452bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff continue; 20462bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff 204720ebf8f15df48f27ce2bf7c1b3ae87565235600fSteve Naroff case OPC_CheckSame: 204820ebf8f15df48f27ce2bf7c1b3ae87565235600fSteve Naroff if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break; 204920ebf8f15df48f27ce2bf7c1b3ae87565235600fSteve Naroff continue; 205020ebf8f15df48f27ce2bf7c1b3ae87565235600fSteve Naroff case OPC_CheckPatternPredicate: 20511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break; 20524ba0acbe835b12b3755d138f364cd67044d7322fSteve Naroff continue; 20534ba0acbe835b12b3755d138f364cd67044d7322fSteve Naroff case OPC_CheckPredicate: 20544ba0acbe835b12b3755d138f364cd67044d7322fSteve Naroff if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this, 2055d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer N.getNode())) 20561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump break; 20572bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff continue; 20582bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff case OPC_CheckComplexPat: { 20592bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff unsigned CPNum = MatcherTable[MatcherIndex++]; 2060d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer unsigned RecNo = MatcherTable[MatcherIndex++]; 20612bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat"); 20622bd03923398add1bcb10d40c283cb0eb8ade74daSteve Naroff if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo], CPNum, 2063909f02a69786e94d34d34fc8d4ea3e160bcff775Fariborz Jahanian RecordedNodes)) 2064b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff break; 206501c5748c29e75b29cab5fc7d8ad1b173b29c7ecfChris Lattner continue; 206601c5748c29e75b29cab5fc7d8ad1b173b29c7ecfChris Lattner } 206785f9bceab1542aafff012d4d28e998f4ba16e362Anders Carlsson case OPC_CheckOpcode: 20680d504c1da852e58ff802545c823ecff3b6c654b8Daniel Dunbar if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break; 20692085fd6cd22ec5c268175251db10d7c60caf7aaaChris Lattner continue; 20702085fd6cd22ec5c268175251db10d7c60caf7aaaChris Lattner 20712085fd6cd22ec5c268175251db10d7c60caf7aaaChris Lattner case OPC_CheckType: 2072dcbc5b0b0722282a0fdd829359fe0d7e22adb882Chris Lattner if (!::CheckType(MatcherTable, MatcherIndex, N, TLI)) break; 20731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump continue; 20740750618b0beacdc9b0a9e210a661e4746823ced7Chris Lattner 20754c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff case OPC_SwitchOpcode: { 2076e64b7771dca65d737dfc49b6137bd69fc1ff6bd2Chris Lattner unsigned CurNodeOpcode = N.getOpcode(); 2077311ff02fae0392bee6abe7723cdf5a69b2899a47Chris Lattner unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; 2078311ff02fae0392bee6abe7723cdf5a69b2899a47Chris Lattner unsigned CaseSize; 2079b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff while (1) { 20801a9376408941d65c285ddad8a0f1a2ac518df344Steve Naroff // Get the size of this case. 20811a9376408941d65c285ddad8a0f1a2ac518df344Steve Naroff CaseSize = MatcherTable[MatcherIndex++]; 2082b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff if (CaseSize & 128) 2083b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); 2084b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff if (CaseSize == 0) break; 2085b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff 20861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump uint16_t Opc = MatcherTable[MatcherIndex++]; 20876e94ef5696cfb005d3fc7bbac8dcf7690b64f0a5Ted Kremenek Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; 2088077bf5e2f48acfa9e7d69429b6e4ba86ea14896dChris Lattner 2089726e168dc09fb23f53c7b004f8e919421ee91806Chris Lattner // If the opcode matches, then we will execute this case. 2090b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff if (CurNodeOpcode == Opc) 2091b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff break; 2092dcbc5b0b0722282a0fdd829359fe0d7e22adb882Chris Lattner 20934c3580e5f9d1804fa08fecca76ad5089bc9965feSteve Naroff // Otherwise, skip over this case. 2094b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff MatcherIndex += CaseSize; 2095b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff } 2096b42f8415bcfb84c208fd577458ce1bbc2cd800feSteve Naroff 2097b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff // If no cases matched, bail out. 20981d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian if (CaseSize == 0) break; 20991d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian 2100ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff // Otherwise, execute the case we found. 2101934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroff DEBUG(errs() << " OpcodeSwitch from " << SwitchStart 21021eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump << " to " << MatcherIndex << "\n"); 2103ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff continue; 21048189cde56b4f6f938cd65f53c932fe1860d0204cTed Kremenek } 21051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2106ebf2b56bce1ea6b1b8133c1f0e2131503d229d2dSteve Naroff case OPC_SwitchType: { 2107f04da137288c48879a86e9e1d4014db8e28dbae1Chris Lattner MVT::SimpleValueType CurNodeVT = N.getValueType().getSimpleVT().SimpleTy; 210888465d3e996e627bbaa11099b039ddab66d5af2cAnders Carlsson unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; 210988465d3e996e627bbaa11099b039ddab66d5af2cAnders Carlsson unsigned CaseSize; 2110f1b48b7014992155286d58bb1676f9f51031d18bAnders Carlsson while (1) { 2111906082edf2aea1c6de2926f93a8d7121e49d2a54Sebastian Redl // Get the size of this case. 21121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump CaseSize = MatcherTable[MatcherIndex++]; 2113183700f494ec9b6701b6efe82bcb25f4c79ba561John McCall if (CaseSize & 128) 21141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); 21151d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian if (CaseSize == 0) break; 21165291c3cec0dbe8ad1d8e7e67e93af2b1586d5400Douglas Gregor 21175291c3cec0dbe8ad1d8e7e67e93af2b1586d5400Douglas Gregor MVT::SimpleValueType CaseVT = 21181d35b168348a308696d1fce29a18c14143f61d6aFariborz Jahanian (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 2119934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroff if (CaseVT == MVT::iPTR) 2120934f276cc5b45e19cd12ebb2d04fd7972a23865cSteve Naroff CaseVT = TLI.getPointerTy().SimpleTy; 2121d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2122d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff // If the VT matches, then we will execute this case. 2123d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff if (CurNodeVT == CaseVT) 2124d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff break; 2125d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2126d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff // Otherwise, skip over this case. 212732174826f58d38c20f376fe64ceac4a278133883Steve Naroff MatcherIndex += CaseSize; 212832174826f58d38c20f376fe64ceac4a278133883Steve Naroff } 212932174826f58d38c20f376fe64ceac4a278133883Steve Naroff 213032174826f58d38c20f376fe64ceac4a278133883Steve Naroff // If no cases matched, bail out. 213132174826f58d38c20f376fe64ceac4a278133883Steve Naroff if (CaseSize == 0) break; 2132d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2133d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff // Otherwise, execute the case we found. 2134d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff DEBUG(errs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString() 2135d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff << "] from " << SwitchStart << " to " << MatcherIndex<<'\n'); 2136d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff continue; 2137d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff } 213861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_CheckChild0Type: case OPC_CheckChild1Type: 213961477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_CheckChild2Type: case OPC_CheckChild3Type: 214061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_CheckChild4Type: case OPC_CheckChild5Type: 214161477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_CheckChild6Type: case OPC_CheckChild7Type: 214261477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI, 214361477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian Opcode-OPC_CheckChild0Type)) 214461477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian break; 214561477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian continue; 214661477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_CheckCondCode: 214761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break; 214861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian continue; 2149291e04b29669d6576e19e35555bbc3cadd2f6649Fariborz Jahanian case OPC_CheckValueType: 2150b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI)) break; 215132132a08c0aecc316728d81e1d93726707fde14bFariborz Jahanian continue; 215232132a08c0aecc316728d81e1d93726707fde14bFariborz Jahanian case OPC_CheckInteger: 215384aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break; 215484aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian continue; 215584aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian case OPC_CheckAndImm: 215684aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break; 215784aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian continue; 215884aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian case OPC_CheckOrImm: 215984aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break; 216084aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian continue; 216184aa9464a4542b7b63d3b1f9953f9169ce536527Fariborz Jahanian 2162d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff case OPC_CheckFoldableChainNode: { 2163d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff assert(NodeStack.size() != 1 && "No parent node"); 21644f95b750534f2111f28434b282bcbd5656002816Steve Naroff // Verify that all intermediate nodes between the root and this one have 21654f95b750534f2111f28434b282bcbd5656002816Steve Naroff // a single use. 21664f95b750534f2111f28434b282bcbd5656002816Steve Naroff bool HasMultipleUses = false; 2167cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) 21681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (!NodeStack[i].hasOneUse()) { 2169cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff HasMultipleUses = true; 2170cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff break; 2171cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff } 2172cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff if (HasMultipleUses) break; 2173cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff 2174cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff // Check to see that the target thinks this is profitable to fold and that 2175cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff // we can fold it without inducing cycles in the graph. 2176cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(), 2177cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff NodeToMatch) || 2178cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(), 2179cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff NodeToMatch, OptLevel, 21804f95b750534f2111f28434b282bcbd5656002816Steve Naroff true/*We validate our own chains*/)) 2181cda658e90d7ffe0ece23d741ff82cf764108f66cSteve Naroff break; 21824f95b750534f2111f28434b282bcbd5656002816Steve Naroff 21834f95b750534f2111f28434b282bcbd5656002816Steve Naroff continue; 21844f95b750534f2111f28434b282bcbd5656002816Steve Naroff } 21854f95b750534f2111f28434b282bcbd5656002816Steve Naroff case OPC_EmitInteger: { 21864f95b750534f2111f28434b282bcbd5656002816Steve Naroff MVT::SimpleValueType VT = 21874f95b750534f2111f28434b282bcbd5656002816Steve Naroff (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 2188d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer int64_t Val = MatcherTable[MatcherIndex++]; 2189d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer if (Val & 128) 21904f95b750534f2111f28434b282bcbd5656002816Steve Naroff Val = GetVBR(Val, MatcherTable, MatcherIndex); 21914f95b750534f2111f28434b282bcbd5656002816Steve Naroff RecordedNodes.push_back(CurDAG->getTargetConstant(Val, VT)); 21924f95b750534f2111f28434b282bcbd5656002816Steve Naroff continue; 21934f95b750534f2111f28434b282bcbd5656002816Steve Naroff } 2194b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff case OPC_EmitRegister: { 219561477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian MVT::SimpleValueType VT = 219661477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 219772564e73277e29f6db3305d1f27ba408abb7ed88Douglas Gregor unsigned RegNo = MatcherTable[MatcherIndex++]; 219861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian RecordedNodes.push_back(CurDAG->getRegister(RegNo, VT)); 219961477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian continue; 220061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian } 220161477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian 220261477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_EmitConvertToTarget: { 220361477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian // Convert from IMM/FPIMM to target version. 220461477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian unsigned RecNo = MatcherTable[MatcherIndex++]; 220561477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 2206183700f494ec9b6701b6efe82bcb25f4c79ba561John McCall SDValue Imm = RecordedNodes[RecNo]; 220761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian 220872564e73277e29f6db3305d1f27ba408abb7ed88Douglas Gregor if (Imm->getOpcode() == ISD::Constant) { 220961477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian int64_t Val = cast<ConstantSDNode>(Imm)->getZExtValue(); 221061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian Imm = CurDAG->getTargetConstant(Val, Imm.getValueType()); 221161477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian } else if (Imm->getOpcode() == ISD::ConstantFP) { 221261477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue(); 22133d7e7865bd0863fcf36aee14d8911b785dde57ddSteve Naroff Imm = CurDAG->getTargetConstantFP(*Val, Imm.getValueType()); 22143d7e7865bd0863fcf36aee14d8911b785dde57ddSteve Naroff } 22153d7e7865bd0863fcf36aee14d8911b785dde57ddSteve Naroff 22163d7e7865bd0863fcf36aee14d8911b785dde57ddSteve Naroff RecordedNodes.push_back(Imm); 221761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian continue; 2218e66894c3dea9aeab4403e67bf5baabc20da61cb3Fariborz Jahanian } 22191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 222061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0 2221d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff case OPC_EmitMergeInputChains1_1: { // OPC_EmitMergeInputChains, 1, 1 22221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // These are space-optimized forms of OPC_EmitMergeInputChains. 2223d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff assert(InputChain.getNode() == 0 && 2224d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff "EmitMergeInputChains should be the first chain producing node"); 22256cafbf2a678c95d4cdc09ecaa782319bbcf56944Steve Naroff assert(ChainNodesMatched.empty() && 2226d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff "Should only have one EmitMergeInputChains per match"); 2227d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2228d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff // Read all of the chained nodes. 2229d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff unsigned RecNo = Opcode == OPC_EmitMergeInputChains1_1; 2230d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 2231d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff ChainNodesMatched.push_back(RecordedNodes[RecNo].getNode()); 2232d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2233d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // FIXME: What if other value results of the node have uses not matched 2234d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // by this pattern? 2235d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff if (ChainNodesMatched.back() != NodeToMatch && 2236d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff !RecordedNodes[RecNo].hasOneUse()) { 223761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian ChainNodesMatched.clear(); 223861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian break; 22399165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff } 224061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian 224161477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian // Merge the input chains if they are not intra-pattern references. 22429165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); 2243d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff 2244d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff if (InputChain.getNode() == 0) 22451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump break; // Failed to merge. 2246d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff continue; 224761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian } 224861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian 2249d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff case OPC_EmitMergeInputChains: { 2250d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff assert(InputChain.getNode() == 0 && 2251d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff "EmitMergeInputChains should be the first chain producing node"); 22521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // This node gets a list of nodes we matched in the input that have 2253291e04b29669d6576e19e35555bbc3cadd2f6649Fariborz Jahanian // chains. We want to token factor all of the input chains to these nodes 22541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // together. However, if any of the input chains is actually one of the 2255291e04b29669d6576e19e35555bbc3cadd2f6649Fariborz Jahanian // nodes matched in this pattern, then we have an intra-match reference. 2256d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff // Ignore these because the newly token factored chain should not refer to 2257d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer // the old nodes. 2258d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer unsigned NumChains = MatcherTable[MatcherIndex++]; 2259d5255f55ebe8803ee10b01589a64156e5e9cde5eSteve Naroff assert(NumChains != 0 && "Can't TF zero chains"); 226061477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian 226161477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian assert(ChainNodesMatched.empty() && 226261477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian "Should only have one EmitMergeInputChains per match"); 2263aba49d10daaedbc9e8bf6f4c6bfbfa6ea668baa6Steve Naroff 2264aba49d10daaedbc9e8bf6f4c6bfbfa6ea668baa6Steve Naroff // Read all of the chained nodes. 2265aba49d10daaedbc9e8bf6f4c6bfbfa6ea668baa6Steve Naroff for (unsigned i = 0; i != NumChains; ++i) { 226661477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian unsigned RecNo = MatcherTable[MatcherIndex++]; 226761477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 226861477f7a4ccd3ac785afd9708d77a53869a53a76Fariborz Jahanian ChainNodesMatched.push_back(RecordedNodes[RecNo].getNode()); 22699165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff 22709165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff // FIXME: What if other value results of the node have uses not matched 22719165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff // by this pattern? 22724c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian if (ChainNodesMatched.back() != NodeToMatch && 22734c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian !RecordedNodes[RecNo].hasOneUse()) { 22744c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian ChainNodesMatched.clear(); 22754c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian break; 22764c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian } 22774c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian } 22784c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22794c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian // If the inner loop broke out, the match fails. 22804c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian if (ChainNodesMatched.empty()) 22814c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian break; 22824c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22834c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian // Merge the input chains if they are not intra-pattern references. 2284fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); 22854c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22864c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian if (InputChain.getNode() == 0) 22874c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian break; // Failed to merge. 22884c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22894c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian continue; 22904c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian } 22914c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22924c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian case OPC_EmitCopyToReg: { 22934c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian unsigned RecNo = MatcherTable[MatcherIndex++]; 22944c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 22954c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian unsigned DestPhysReg = MatcherTable[MatcherIndex++]; 22964c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 22974c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian if (InputChain.getNode() == 0) 2298d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer InputChain = CurDAG->getEntryNode(); 22994c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 23004c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian InputChain = CurDAG->getCopyToReg(InputChain, NodeToMatch->getDebugLoc(), 23014c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian DestPhysReg, RecordedNodes[RecNo], 23024c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian InputFlag); 23034c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 2304d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer InputFlag = InputChain.getValue(1); 23054c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian continue; 23064c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian } 23074c863ef92c2b74572090da245c87e1487b0b596cFariborz Jahanian 2308a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian case OPC_EmitNodeXForm: { 2309b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff unsigned XFormNo = MatcherTable[MatcherIndex++]; 2310a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian unsigned RecNo = MatcherTable[MatcherIndex++]; 2311a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 23120953e767ff7817f97b3ab20896b229891eeff45bJohn McCall RecordedNodes.push_back(RunSDNodeXForm(RecordedNodes[RecNo], XFormNo)); 2313a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek continue; 2314ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor } 2315ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor 2316264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola case OPC_EmitNode: 2317264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola case OPC_MorphNodeTo: { 2318ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis uint16_t TargetOpc = MatcherTable[MatcherIndex++]; 23191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; 2320a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis unsigned EmitNodeInfo = MatcherTable[MatcherIndex++]; 232116573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor // Get the result VT list. 232216573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor unsigned NumVTs = MatcherTable[MatcherIndex++]; 2323a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian SmallVector<EVT, 4> VTs; 2324a70711b296953bfe2fc17351f3908dcc417c404aFariborz Jahanian for (unsigned i = 0; i != NumVTs; ++i) { 2325b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff MVT::SimpleValueType VT = 232609b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; 232751efe56116a6283ba707cfb8bf8b4e0b13cb41acDouglas Gregor if (VT == MVT::iPTR) VT = TLI.getPointerTy().SimpleTy; 232851efe56116a6283ba707cfb8bf8b4e0b13cb41acDouglas Gregor VTs.push_back(VT); 232909b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff } 23309165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff 23319165ad378f8d25eb5c378a8e9540089afce421ffSteve Naroff if (EmitNodeInfo & OPFL_Chain) 2332a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek VTs.push_back(MVT::Other); 233309b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff if (EmitNodeInfo & OPFL_FlagOutput) 233409b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff VTs.push_back(MVT::Flag); 2335fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar 2336fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar // This is hot code, so optimize the two most common cases of 1 and 2 233752b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian // results. 233852b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian SDVTList VTList; 233952b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian if (VTs.size() == 1) 234052b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian VTList = CurDAG->getVTList(VTs[0]); 234152b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian else if (VTs.size() == 2) 234252b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian VTList = CurDAG->getVTList(VTs[0], VTs[1]); 234352b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian else 234452b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian VTList = CurDAG->getVTList(VTs.data(), VTs.size()); 234552b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian 234652b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian // Get the operand list. 234752b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian unsigned NumOps = MatcherTable[MatcherIndex++]; 2348e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian SmallVector<SDValue, 8> Ops; 2349fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar for (unsigned i = 0; i != NumOps; ++i) { 2350fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar unsigned RecNo = MatcherTable[MatcherIndex++]; 2351e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian if (RecNo & 128) 2352fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); 2353e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian 2354e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian assert(RecNo < RecordedNodes.size() && "Invalid EmitNode"); 2355e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian Ops.push_back(RecordedNodes[RecNo]); 2356e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian } 2357e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian 2358e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian // If there are variadic operands to add, handle them now. 2359e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian if (EmitNodeInfo & OPFL_VariadicInfo) { 2360e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian // Determine the start index to copy from. 2361e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo); 2362e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0; 2363e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian assert(NodeToMatch->getNumOperands() >= FirstOpToCopy && 2364e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian "Invalid variadic node"); 2365e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian // Copy all of the variadic operands, not including a potential flag 2366e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian // input. 2367e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands(); 2368e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian i != e; ++i) { 2369e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian SDValue V = NodeToMatch->getOperand(i); 2370e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian if (V.getValueType() == MVT::Flag) break; 2371e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian Ops.push_back(V); 2372e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian } 2373e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian } 2374e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian 2375e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian // If this has chain/flag inputs, add them. 2376e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian if (EmitNodeInfo & OPFL_Chain) 2377e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian Ops.push_back(InputChain); 2378e8c28dffce846b1783fd11cc130b4be4f9894301Fariborz Jahanian if ((EmitNodeInfo & OPFL_FlagInput) && InputFlag.getNode() != 0) 2379abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian Ops.push_back(InputFlag); 2380abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian 2381abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // Create the node. 2382abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian SDNode *Res = 0; 2383abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian if (Opcode != OPC_MorphNodeTo) { 2384abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // If this is a normal EmitNode command, just create the new node and 2385abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // add the results to the RecordedNodes list. 2386fa297fb29b38991c537a0ae90ff595102dcd21a9Daniel Dunbar Res = CurDAG->getMachineNode(TargetOpc, NodeToMatch->getDebugLoc(), 2387abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian VTList, Ops.data(), Ops.size()); 2388abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian 2389abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // Add all the non-flag/non-chain results to the RecordedNodes list. 2390abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian for (unsigned i = 0, e = VTs.size(); i != e; ++i) { 2391abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian if (VTs[i] == MVT::Other || VTs[i] == MVT::Flag) break; 2392abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian RecordedNodes.push_back(SDValue(Res, i)); 239352b2e1e0775a7a50616b02be4af92295c61e37f5Fariborz Jahanian } 2394abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian 2395abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian } else { 2396abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops.data(), Ops.size(), 2397abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian EmitNodeInfo); 2398d999b3736ce2646ae0711416b421d906298764dbBenjamin Kramer } 2399abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian 2400abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // If the node had chain/flag results, update our notion of the current 2401abfd83e74ca8a7553e375dd4631d2570f33648b4Fariborz Jahanian // chain and flag. 2402c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff if (EmitNodeInfo & OPFL_FlagOutput) { 2403b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff InputFlag = SDValue(Res, VTs.size()-1); 2404c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff if (EmitNodeInfo & OPFL_Chain) 2405c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff InputChain = SDValue(Res, VTs.size()-2); 240646a98a7df016b4a6835a1902c2f8ff2a3a8012baSteve Naroff } else if (EmitNodeInfo & OPFL_Chain) 2407c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff InputChain = SDValue(Res, VTs.size()-1); 2408c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff 2409c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff // If the OPFL_MemRefs flag is set on this node, slap all of the 2410c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff // accumulated memrefs onto it. 2411c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff // 2412c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff // FIXME: This is vastly incorrect for patterns with multiple outputs 2413c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff // instructions that access memory and for ComplexPatterns that match 2414ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor // loads. 2415264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola if (EmitNodeInfo & OPFL_MemRefs) { 2416264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola MachineSDNode::mmo_iterator MemRefs = 2417ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis MF->allocateMemRefsArray(MatchedMemRefs.size()); 24181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump std::copy(MatchedMemRefs.begin(), MatchedMemRefs.end(), MemRefs); 2419a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis cast<MachineSDNode>(Res) 242016573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor ->setMemRefs(MemRefs, MemRefs + MatchedMemRefs.size()); 242116573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor } 2422c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff 2423c0a123c81b4c88eb713ad2f82e3d7ad0e9ef7c68Steve Naroff DEBUG(errs() << " " 242409b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff << (Opcode == OPC_MorphNodeTo ? "Morphed" : "Created") 2425b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff << " node: "; Res->dump(CurDAG); errs() << "\n"); 242609b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff 242709b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff // If this was a MorphNodeTo then we're completely done! 2428a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (Opcode == OPC_MorphNodeTo) { 242909b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff // Update chain and flag uses. 243009b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff UpdateChainsAndFlags(NodeToMatch, InputChain, ChainNodesMatched, 2431a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek InputFlag, FlagResultNodesMatched, true); 243209b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff return Res; 243309b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff } 2434a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 243509b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff continue; 2436ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor } 2437264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola 2438264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola case OPC_MarkFlagResults: { 2439ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis unsigned NumNodes = MatcherTable[MatcherIndex++]; 24400ed844b04ea4387caa4e1cf3dc375d269657536bChris Lattner 2441a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis // Read and remember all the flag-result nodes. 244216573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor for (unsigned i = 0; i != NumNodes; ++i) { 244316573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor unsigned RecNo = MatcherTable[MatcherIndex++]; 244409b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff if (RecNo & 128) 244509b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); 2446874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff 2447b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff assert(RecNo < RecordedNodes.size() && "Invalid CheckSame"); 2448874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff FlagResultNodesMatched.push_back(RecordedNodes[RecNo].getNode()); 2449874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff } 2450465d41b92b2c862f3062c412a0538db65c6a2661Abramo Bagnara continue; 24510ed844b04ea4387caa4e1cf3dc375d269657536bChris Lattner } 2452df042e6c2bf06b2d9ed53c52469599ac1bd93a3fTed Kremenek 2453874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff case OPC_CompleteMatch: { 2454874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff // The match has been completed, and any new nodes (if any) have been 2455874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff // created. Patch up references to the matched dag to use the newly 2456a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek // created nodes. 2457874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff unsigned NumResults = MatcherTable[MatcherIndex++]; 2458874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff 2459a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek for (unsigned i = 0; i != NumResults; ++i) { 2460874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff unsigned ResSlot = MatcherTable[MatcherIndex++]; 2461ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor if (ResSlot & 128) 2462264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex); 2463264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola 2464ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis assert(ResSlot < RecordedNodes.size() && "Invalid CheckSame"); 24651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump SDValue Res = RecordedNodes[ResSlot]; 2466a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis 246716573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor assert(i < NodeToMatch->getNumValues() && 246816573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor NodeToMatch->getValueType(i) != MVT::Other && 2469874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff NodeToMatch->getValueType(i) != MVT::Flag && 2470874e232a0d5e179197de860e6dfa3e99cd42ad30Steve Naroff "Invalid number of results to complete!"); 247180a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian assert((NodeToMatch->getValueType(i) == Res.getValueType() || 2472b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff NodeToMatch->getValueType(i) == MVT::iPTR || 247380a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian Res.getValueType() == MVT::iPTR || 247480a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian NodeToMatch->getValueType(i).getSizeInBits() == 2475a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek Res.getValueType().getSizeInBits()) && 247680a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian "invalid replacement"); 247780a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res); 2478a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek } 247980a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian 248080a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian // If the root node defines a flag, add it to the flag nodes to update 2481a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek // list. 248280a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian if (NodeToMatch->getValueType(NodeToMatch->getNumValues()-1) == MVT::Flag) 2483ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor FlagResultNodesMatched.push_back(NodeToMatch); 2484264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola 2485264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola // Update chain and flag uses. 2486ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis UpdateChainsAndFlags(NodeToMatch, InputChain, ChainNodesMatched, 24871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InputFlag, FlagResultNodesMatched, false); 2488a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis 248916573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor assert(NodeToMatch->use_empty() && 249016573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor "Didn't replace all uses of the node?"); 249180a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian 249280a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian // FIXME: We just return here, which interacts correctly with SelectRoot 24931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // above. We should fix this to not return an SDNode* anymore. 249480a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian return 0; 2495b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff } 24961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 249780a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian 249880a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian // If the code reached this point, then the match failed. See if there is 2499465d41b92b2c862f3062c412a0538db65c6a2661Abramo Bagnara // another child to try in the current 'Scope', otherwise pop it until we 25000ed844b04ea4387caa4e1cf3dc375d269657536bChris Lattner // find a case to check. 2501df042e6c2bf06b2d9ed53c52469599ac1bd93a3fTed Kremenek DEBUG(errs() << " Match failed at index " << CurrentOpcodeIndex << "\n"); 250280a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian ++NumDAGIselRetries; 250380a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian while (1) { 250480a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian if (MatchScopes.empty()) { 2505a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek CannotYetSelect(NodeToMatch); 250680a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian return 0; 250780a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian } 2508a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek 250980a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian // Restore the interpreter state back to the point where the scope was 2510ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor // formed. 2511264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola MatchScope &LastScope = MatchScopes.back(); 2512264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola RecordedNodes.resize(LastScope.NumRecordedNodes); 2513ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis NodeStack.clear(); 25141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end()); 2515a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis N = NodeStack.back(); 251616573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor 251716573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size()) 251880a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian MatchedMemRefs.resize(LastScope.NumMatchedMemRefs); 251980a6a5abbbf0b7c45f535d4e0b0d00f4c3f320eaFariborz Jahanian MatcherIndex = LastScope.FailIndex; 25201284db8d85782c4eb1d41eb02358613c8681f07fSteve Naroff 2521b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff DEBUG(errs() << " Continuing at " << MatcherIndex << "\n"); 2522acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian 2523acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian InputChain = LastScope.InputChain; 2524a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek InputFlag = LastScope.InputFlag; 2525acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian if (!LastScope.HasChainNodesMatched) 2526acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian ChainNodesMatched.clear(); 2527a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek if (!LastScope.HasFlagResultNodesMatched) 2528acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian FlagResultNodesMatched.clear(); 2529acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian 25301284db8d85782c4eb1d41eb02358613c8681f07fSteve Naroff // Check to see what the offset is at the new MatcherIndex. If it is zero 2531acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian // we have reached the end of this scope, otherwise we have another child 2532ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor // in the current scope to try. 2533264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola unsigned NumToSkip = MatcherTable[MatcherIndex++]; 2534264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola if (NumToSkip & 128) 2535ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex); 25361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2537a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis // If we have another child in this scope to match, update FailIndex and 253816573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor // try it. 253916573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor if (NumToSkip != 0) { 2540acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian LastScope.FailIndex = MatcherIndex+NumToSkip; 2541acb4977dd3b6a678bfbdb80781bfe3c617be3f24Fariborz Jahanian break; 254209b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff } 2543b29b427fa977e8e13ea104d22b193b2cd8a4a52fSteve Naroff 254409b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff // End of this scope, pop it and try the next child in the containing 254509b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff // scope. 25460953e767ff7817f97b3ab20896b229891eeff45bJohn McCall MatchScopes.pop_back(); 2547a526c5c67e5a0473c340903ee542ce570119665fTed Kremenek } 254809b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff } 2549ce056bcaa1c97b89a4b2de2112c62d060863be2bDouglas Gregor} 2550264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola 2551264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola 2552ef177820100ab583b08fd3056e2a5a52ee4b1629Argyrios Kyrtzidis 25531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpvoid SelectionDAGISel::CannotYetSelect(SDNode *N) { 2554a1d5662d96465f0fddf8819d245da4d19b892effArgyrios Kyrtzidis std::string msg; 255516573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor raw_string_ostream Msg(msg); 255616573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor Msg << "Cannot yet select: "; 255709b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff 255809b266eb2a014b7af4dc06126c47b7629030ce09Steve Naroff if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN && 2559d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian N->getOpcode() != ISD::INTRINSIC_WO_CHAIN && 2560d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian N->getOpcode() != ISD::INTRINSIC_VOID) { 2561d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian N->printrFull(Msg, CurDAG); 2562d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian } else { 2563d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other; 2564d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian unsigned iid = 2565d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue(); 2566d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian if (iid < Intrinsic::num_intrinsics) 2567d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid); 2568264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo()) 2569264ba48dc98f3f843935a485d5b086f7e0fdc4f1Rafael Espindola Msg << "target intrinsic %" << TII->getName(iid); 2570d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian else 257116573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor Msg << "unknown intrinsic #" << iid; 257216573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor } 257316573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor report_fatal_error(Msg.str()); 257416573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor} 257516573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregor 257616573fa9705b546b7597c273b25b85d6321e2b33Douglas Gregorchar SelectionDAGISel::ID = 0; 2577d314e9e12326c2fd8f140adc4c769d13b483b3f6Fariborz Jahanian