X86ISelLowering.cpp revision 8968450305c28444edc3c272d8752a8db0c2f34a
1f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// 3f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// The LLVM Compiler Infrastructure 4f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// 5f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// This file is distributed under the University of Illinois Open Source 6f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// License. See LICENSE.TXT for details. 7f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// 8f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette//===----------------------------------------------------------------------===// 9f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// 10f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// This file defines the interfaces that X86 uses to lower LLVM code into a 11f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// selection DAG. 12f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// 13f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette//===----------------------------------------------------------------------===// 14f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 15f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "X86.h" 16f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "X86InstrBuilder.h" 17f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "X86ISelLowering.h" 18f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "X86MachineFunctionInfo.h" 19f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "X86TargetMachine.h" 20f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CallingConv.h" 21f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Constants.h" 22f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/DerivedTypes.h" 23f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/GlobalVariable.h" 24f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Function.h" 25f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Intrinsics.h" 26f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/ADT/BitVector.h" 27f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/ADT/VectorExtras.h" 28f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/CallingConvLower.h" 29f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/MachineFrameInfo.h" 30f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/MachineFunction.h" 31f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/MachineInstrBuilder.h" 32f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/MachineModuleInfo.h" 33f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/MachineRegisterInfo.h" 34f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/PseudoSourceValue.h" 35f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/CodeGen/SelectionDAG.h" 36f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Support/MathExtras.h" 37f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Support/Debug.h" 38f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/Target/TargetOptions.h" 39f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/ADT/SmallSet.h" 40f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette#include "llvm/ADT/StringExtras.h" 41f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viveretteusing namespace llvm; 42f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 43f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette// Forward declarations. 44f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverettestatic SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 46f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan ViveretteX86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette : TargetLowering(TM) { 48f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette Subtarget = &TM.getSubtarget<X86Subtarget>(); 49f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette X86ScalarSSEf64 = Subtarget->hasSSE2(); 50f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette X86ScalarSSEf32 = Subtarget->hasSSE1(); 51f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 53f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette bool Fast = false; 54f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 55f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette RegInfo = TM.getRegisterInfo(); 56f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 57f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // Set up the TargetLowering object. 58f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 59f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // X86 is weird, it always uses i8 for shift amounts and setcc results. 60f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setShiftAmountType(MVT::i8); 61f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setSetCCResultContents(ZeroOrOneSetCCResult); 62f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setSchedulingPreference(SchedulingForRegPressure); 63f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 64f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setStackPointerRegisterToSaveRestore(X86StackPtr); 65f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 66f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette if (Subtarget->isTargetDarwin()) { 67f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 68f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreSetJmp(false); 69f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreLongJmp(false); 70f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } else if (Subtarget->isTargetMingw()) { 71f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // MS runtime is weird: it exports _setjmp, but longjmp! 72f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreSetJmp(true); 73f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreLongJmp(false); 74f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } else { 75f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreSetJmp(true); 76f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setUseUnderscoreLongJmp(true); 77f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } 78f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 79f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // Set up the register classes. 80f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette addRegisterClass(MVT::i8, X86::GR8RegisterClass); 81f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette addRegisterClass(MVT::i16, X86::GR16RegisterClass); 82f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette addRegisterClass(MVT::i32, X86::GR32RegisterClass); 83f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette if (Subtarget->is64Bit()) 84f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette addRegisterClass(MVT::i64, X86::GR64RegisterClass); 85f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 86f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 87f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 88f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // We don't accept any truncstore of integer registers. 89f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i64, MVT::i32, Expand); 90f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i64, MVT::i16, Expand); 91f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 92f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i32, MVT::i16, Expand); 93f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 94f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setTruncStoreAction(MVT::i16, MVT::i8, Expand); 95f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 96f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 97f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // operation. 98f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 99f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 100f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 101f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 102f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette if (Subtarget->is64Bit()) { 103f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 104f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 105f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } else { 106f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette if (X86ScalarSSEf64) 107f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 108f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 109f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette else 110f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 111f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } 112f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 113f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 114f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // this operation. 115f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 116f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 117f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // SSE has no i16 to fp conversion, only i32 118f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette if (X86ScalarSSEf32) { 119f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 120f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // f32 and f64 cases are Legal, f80 case is not 121f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 122f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } else { 123f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 124f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 125f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette } 126f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette 127f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 128f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette // are Legal, f80 is custom lowered. 129f11879ae94e7598cb6ae59fdc13104947b66e3e6Alan Viverette setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 130 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 131 132 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 133 // this operation. 134 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 135 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 136 137 if (X86ScalarSSEf32) { 138 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 139 // f32 and f64 cases are Legal, f80 case is not 140 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 141 } else { 142 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 143 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 144 } 145 146 // Handle FP_TO_UINT by promoting the destination to a larger signed 147 // conversion. 148 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 149 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 151 152 if (Subtarget->is64Bit()) { 153 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 155 } else { 156 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 157 // Expand FP_TO_UINT into a select. 158 // FIXME: We would like to use a Custom expander here eventually to do 159 // the optimal thing for SSE vs. the default expansion in the legalizer. 160 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 161 else 162 // With SSE3 we can use fisttpll to convert to a signed i64. 163 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 164 } 165 166 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 167 if (!X86ScalarSSEf64) { 168 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 169 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 170 } 171 172 // Scalar integer divide and remainder are lowered to use operations that 173 // produce two results, to match the available instructions. This exposes 174 // the two-result form to trivial CSE, which is able to combine x/y and x%y 175 // into a single instruction. 176 // 177 // Scalar integer multiply-high is also lowered to use two-result 178 // operations, to match the available instructions. However, plain multiply 179 // (low) operations are left as Legal, as there are single-result 180 // instructions for this in x86. Using the two-result multiply instructions 181 // when both high and low results are needed must be arranged by dagcombine. 182 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 183 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 184 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 185 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::SREM , MVT::i8 , Expand); 187 setOperationAction(ISD::UREM , MVT::i8 , Expand); 188 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 189 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 190 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 191 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::SREM , MVT::i16 , Expand); 193 setOperationAction(ISD::UREM , MVT::i16 , Expand); 194 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 195 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 196 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 197 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::SREM , MVT::i32 , Expand); 199 setOperationAction(ISD::UREM , MVT::i32 , Expand); 200 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 201 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 202 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 203 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::SREM , MVT::i64 , Expand); 205 setOperationAction(ISD::UREM , MVT::i64 , Expand); 206 207 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 208 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 209 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 210 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 211 if (Subtarget->is64Bit()) 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 216 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 217 setOperationAction(ISD::FREM , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f64 , Expand); 219 setOperationAction(ISD::FREM , MVT::f80 , Expand); 220 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 221 222 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 223 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 224 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 226 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 227 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 229 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 230 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 231 if (Subtarget->is64Bit()) { 232 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 233 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 234 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 235 } 236 237 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 238 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 239 240 // These should be promoted to a larger select which is supported. 241 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 242 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 243 // X86 wants to expand cmov itself. 244 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 245 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 246 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 249 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 252 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 255 if (Subtarget->is64Bit()) { 256 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 257 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 258 } 259 // X86 ret instruction may pop stack. 260 setOperationAction(ISD::RET , MVT::Other, Custom); 261 if (!Subtarget->is64Bit()) 262 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 263 264 // Darwin ABI issue. 265 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 266 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 268 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 269 if (Subtarget->is64Bit()) 270 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 271 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 272 if (Subtarget->is64Bit()) { 273 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 274 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 275 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 276 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 277 } 278 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 279 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 281 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 282 if (Subtarget->is64Bit()) { 283 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 285 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 286 } 287 288 if (Subtarget->hasSSE1()) 289 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 290 291 if (!Subtarget->hasSSE2()) 292 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 293 294 // Expand certain atomics 295 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i8, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i16, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i32, Custom); 298 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i64, Custom); 299 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i32, Expand); 300 301 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 302 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 303 // FIXME - use subtarget debug flags 304 if (!Subtarget->isTargetDarwin() && 305 !Subtarget->isTargetELF() && 306 !Subtarget->isTargetCygMing()) { 307 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 308 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 309 } 310 311 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 312 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 313 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 314 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 315 if (Subtarget->is64Bit()) { 316 // FIXME: Verify 317 setExceptionPointerRegister(X86::RAX); 318 setExceptionSelectorRegister(X86::RDX); 319 } else { 320 setExceptionPointerRegister(X86::EAX); 321 setExceptionSelectorRegister(X86::EDX); 322 } 323 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 324 325 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 326 327 setOperationAction(ISD::TRAP, MVT::Other, Legal); 328 329 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 330 setOperationAction(ISD::VASTART , MVT::Other, Custom); 331 setOperationAction(ISD::VAEND , MVT::Other, Expand); 332 if (Subtarget->is64Bit()) { 333 setOperationAction(ISD::VAARG , MVT::Other, Custom); 334 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 335 } else { 336 setOperationAction(ISD::VAARG , MVT::Other, Expand); 337 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 338 } 339 340 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 341 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 342 if (Subtarget->is64Bit()) 343 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 344 if (Subtarget->isTargetCygMing()) 345 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 346 else 347 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 348 349 if (X86ScalarSSEf64) { 350 // f32 and f64 use SSE. 351 // Set up the FP register classes. 352 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 353 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 354 355 // Use ANDPD to simulate FABS. 356 setOperationAction(ISD::FABS , MVT::f64, Custom); 357 setOperationAction(ISD::FABS , MVT::f32, Custom); 358 359 // Use XORP to simulate FNEG. 360 setOperationAction(ISD::FNEG , MVT::f64, Custom); 361 setOperationAction(ISD::FNEG , MVT::f32, Custom); 362 363 // Use ANDPD and ORPD to simulate FCOPYSIGN. 364 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 365 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 366 367 // We don't support sin/cos/fmod 368 setOperationAction(ISD::FSIN , MVT::f64, Expand); 369 setOperationAction(ISD::FCOS , MVT::f64, Expand); 370 setOperationAction(ISD::FSIN , MVT::f32, Expand); 371 setOperationAction(ISD::FCOS , MVT::f32, Expand); 372 373 // Expand FP immediates into loads from the stack, except for the special 374 // cases we handle. 375 addLegalFPImmediate(APFloat(+0.0)); // xorpd 376 addLegalFPImmediate(APFloat(+0.0f)); // xorps 377 378 // Floating truncations from f80 and extensions to f80 go through memory. 379 // If optimizing, we lie about this though and handle it in 380 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 381 if (Fast) { 382 setConvertAction(MVT::f32, MVT::f80, Expand); 383 setConvertAction(MVT::f64, MVT::f80, Expand); 384 setConvertAction(MVT::f80, MVT::f32, Expand); 385 setConvertAction(MVT::f80, MVT::f64, Expand); 386 } 387 } else if (X86ScalarSSEf32) { 388 // Use SSE for f32, x87 for f64. 389 // Set up the FP register classes. 390 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 391 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 392 393 // Use ANDPS to simulate FABS. 394 setOperationAction(ISD::FABS , MVT::f32, Custom); 395 396 // Use XORP to simulate FNEG. 397 setOperationAction(ISD::FNEG , MVT::f32, Custom); 398 399 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 400 401 // Use ANDPS and ORPS to simulate FCOPYSIGN. 402 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 403 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 404 405 // We don't support sin/cos/fmod 406 setOperationAction(ISD::FSIN , MVT::f32, Expand); 407 setOperationAction(ISD::FCOS , MVT::f32, Expand); 408 409 // Special cases we handle for FP constants. 410 addLegalFPImmediate(APFloat(+0.0f)); // xorps 411 addLegalFPImmediate(APFloat(+0.0)); // FLD0 412 addLegalFPImmediate(APFloat(+1.0)); // FLD1 413 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 414 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 415 416 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 417 // this though and handle it in InstructionSelectPreprocess so that 418 // dagcombine2 can hack on these. 419 if (Fast) { 420 setConvertAction(MVT::f32, MVT::f64, Expand); 421 setConvertAction(MVT::f32, MVT::f80, Expand); 422 setConvertAction(MVT::f80, MVT::f32, Expand); 423 setConvertAction(MVT::f64, MVT::f32, Expand); 424 // And x87->x87 truncations also. 425 setConvertAction(MVT::f80, MVT::f64, Expand); 426 } 427 428 if (!UnsafeFPMath) { 429 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 430 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 431 } 432 } else { 433 // f32 and f64 in x87. 434 // Set up the FP register classes. 435 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 436 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 437 438 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 439 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 440 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 441 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 442 443 // Floating truncations go through memory. If optimizing, we lie about 444 // this though and handle it in InstructionSelectPreprocess so that 445 // dagcombine2 can hack on these. 446 if (Fast) { 447 setConvertAction(MVT::f80, MVT::f32, Expand); 448 setConvertAction(MVT::f64, MVT::f32, Expand); 449 setConvertAction(MVT::f80, MVT::f64, Expand); 450 } 451 452 if (!UnsafeFPMath) { 453 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 454 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 455 } 456 addLegalFPImmediate(APFloat(+0.0)); // FLD0 457 addLegalFPImmediate(APFloat(+1.0)); // FLD1 458 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 459 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 460 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 461 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 462 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 463 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 464 } 465 466 // Long double always uses X87. 467 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 468 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 469 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 470 { 471 APFloat TmpFlt(+0.0); 472 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 473 addLegalFPImmediate(TmpFlt); // FLD0 474 TmpFlt.changeSign(); 475 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 476 APFloat TmpFlt2(+1.0); 477 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 478 addLegalFPImmediate(TmpFlt2); // FLD1 479 TmpFlt2.changeSign(); 480 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 481 } 482 483 if (!UnsafeFPMath) { 484 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 485 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 486 } 487 488 // Always use a library call for pow. 489 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 490 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 491 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 492 493 // First set operation action for all vector types to expand. Then we 494 // will selectively turn on ones that can be effectively codegen'd. 495 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 496 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 497 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 498 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 499 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 500 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 501 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 502 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 503 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 504 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 505 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 506 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 507 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 508 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 509 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 510 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 511 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 512 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 513 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 514 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 515 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 516 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 517 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 532 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 533 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 534 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 535 } 536 537 if (Subtarget->hasMMX()) { 538 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 539 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 540 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 541 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 542 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 543 544 // FIXME: add MMX packed arithmetics 545 546 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 547 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 548 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 549 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 550 551 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 552 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 553 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 554 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 555 556 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 557 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 558 559 setOperationAction(ISD::AND, MVT::v8i8, Promote); 560 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 561 setOperationAction(ISD::AND, MVT::v4i16, Promote); 562 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 563 setOperationAction(ISD::AND, MVT::v2i32, Promote); 564 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 565 setOperationAction(ISD::AND, MVT::v1i64, Legal); 566 567 setOperationAction(ISD::OR, MVT::v8i8, Promote); 568 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 569 setOperationAction(ISD::OR, MVT::v4i16, Promote); 570 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 571 setOperationAction(ISD::OR, MVT::v2i32, Promote); 572 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 573 setOperationAction(ISD::OR, MVT::v1i64, Legal); 574 575 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 576 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 577 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 578 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 579 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 580 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 581 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 582 583 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 584 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 585 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 586 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 587 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 588 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 589 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 590 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 591 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 592 593 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 594 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 595 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 596 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 597 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 598 599 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 600 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 601 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 602 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 603 604 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); 605 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 606 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 607 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 608 609 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 610 } 611 612 if (Subtarget->hasSSE1()) { 613 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 614 615 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 616 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 617 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 618 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 619 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 620 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 621 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 622 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 623 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 624 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 625 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 626 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 627 } 628 629 if (Subtarget->hasSSE2()) { 630 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 631 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 632 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 633 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 634 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 635 636 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 637 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 638 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 639 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 640 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 641 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 642 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 643 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 644 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 645 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 646 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 647 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 648 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 649 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 650 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 651 652 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 653 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 654 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 655 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 656 657 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 658 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 659 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 660 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 661 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 662 663 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 664 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 665 MVT VT = (MVT::SimpleValueType)i; 666 // Do not attempt to custom lower non-power-of-2 vectors 667 if (!isPowerOf2_32(VT.getVectorNumElements())) 668 continue; 669 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 670 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 671 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 672 } 673 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 674 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 675 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 676 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 677 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 678 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 679 if (Subtarget->is64Bit()) { 680 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 681 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 682 } 683 684 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 685 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 686 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 687 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 688 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 689 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 690 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 691 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 692 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 693 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 694 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 695 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 696 } 697 698 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 699 700 // Custom lower v2i64 and v2f64 selects. 701 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 702 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 703 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 704 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 705 706 } 707 708 if (Subtarget->hasSSE41()) { 709 // FIXME: Do we need to handle scalar-to-vector here? 710 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 711 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 712 713 // i8 and i16 vectors are custom , because the source register and source 714 // source memory operand types are not the same width. f32 vectors are 715 // custom since the immediate controlling the insert encodes additional 716 // information. 717 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 718 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 719 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 720 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 721 722 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 723 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 724 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 725 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 726 727 if (Subtarget->is64Bit()) { 728 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 729 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 730 } 731 } 732 733 if (Subtarget->hasSSE42()) { 734 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 735 } 736 737 // We want to custom lower some of our intrinsics. 738 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 739 740 // We have target-specific dag combine patterns for the following nodes: 741 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 742 setTargetDAGCombine(ISD::BUILD_VECTOR); 743 setTargetDAGCombine(ISD::SELECT); 744 setTargetDAGCombine(ISD::STORE); 745 746 computeRegisterProperties(); 747 748 // FIXME: These should be based on subtarget info. Plus, the values should 749 // be smaller when we are in optimizing for size mode. 750 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 751 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 752 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 753 allowUnalignedMemoryAccesses = true; // x86 supports it! 754 setPrefLoopAlignment(16); 755} 756 757 758MVT X86TargetLowering::getSetCCResultType(const SDOperand &) const { 759 return MVT::i8; 760} 761 762 763/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 764/// the desired ByVal argument alignment. 765static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 766 if (MaxAlign == 16) 767 return; 768 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 769 if (VTy->getBitWidth() == 128) 770 MaxAlign = 16; 771 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 772 unsigned EltAlign = 0; 773 getMaxByValAlign(ATy->getElementType(), EltAlign); 774 if (EltAlign > MaxAlign) 775 MaxAlign = EltAlign; 776 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 777 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 778 unsigned EltAlign = 0; 779 getMaxByValAlign(STy->getElementType(i), EltAlign); 780 if (EltAlign > MaxAlign) 781 MaxAlign = EltAlign; 782 if (MaxAlign == 16) 783 break; 784 } 785 } 786 return; 787} 788 789/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 790/// function arguments in the caller parameter area. For X86, aggregates 791/// that contain SSE vectors are placed at 16-byte boundaries while the rest 792/// are at 4-byte boundaries. 793unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 794 if (Subtarget->is64Bit()) 795 return getTargetData()->getABITypeAlignment(Ty); 796 unsigned Align = 4; 797 if (Subtarget->hasSSE1()) 798 getMaxByValAlign(Ty, Align); 799 return Align; 800} 801 802/// getOptimalMemOpType - Returns the target specific optimal type for load 803/// and store operations as a result of memset, memcpy, and memmove 804/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 805/// determining it. 806MVT 807X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 808 bool isSrcConst, bool isSrcStr) const { 809 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 810 return MVT::v4i32; 811 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 812 return MVT::v4f32; 813 if (Subtarget->is64Bit() && Size >= 8) 814 return MVT::i64; 815 return MVT::i32; 816} 817 818 819/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 820/// jumptable. 821SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table, 822 SelectionDAG &DAG) const { 823 if (usesGlobalOffsetTable()) 824 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 825 if (!Subtarget->isPICStyleRIPRel()) 826 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 827 return Table; 828} 829 830//===----------------------------------------------------------------------===// 831// Return Value Calling Convention Implementation 832//===----------------------------------------------------------------------===// 833 834#include "X86GenCallingConv.inc" 835 836/// LowerRET - Lower an ISD::RET node. 837SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) { 838 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 839 840 SmallVector<CCValAssign, 16> RVLocs; 841 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 842 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 843 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 844 CCInfo.AnalyzeReturn(Op.Val, RetCC_X86); 845 846 // If this is the first return lowered for this function, add the regs to the 847 // liveout set for the function. 848 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 849 for (unsigned i = 0; i != RVLocs.size(); ++i) 850 if (RVLocs[i].isRegLoc()) 851 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 852 } 853 SDOperand Chain = Op.getOperand(0); 854 855 // Handle tail call return. 856 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 857 if (Chain.getOpcode() == X86ISD::TAILCALL) { 858 SDOperand TailCall = Chain; 859 SDOperand TargetAddress = TailCall.getOperand(1); 860 SDOperand StackAdjustment = TailCall.getOperand(2); 861 assert(((TargetAddress.getOpcode() == ISD::Register && 862 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 863 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 864 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 865 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 866 "Expecting an global address, external symbol, or register"); 867 assert(StackAdjustment.getOpcode() == ISD::Constant && 868 "Expecting a const value"); 869 870 SmallVector<SDOperand,8> Operands; 871 Operands.push_back(Chain.getOperand(0)); 872 Operands.push_back(TargetAddress); 873 Operands.push_back(StackAdjustment); 874 // Copy registers used by the call. Last operand is a flag so it is not 875 // copied. 876 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 877 Operands.push_back(Chain.getOperand(i)); 878 } 879 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 880 Operands.size()); 881 } 882 883 // Regular return. 884 SDOperand Flag; 885 886 SmallVector<SDOperand, 6> RetOps; 887 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 888 // Operand #1 = Bytes To Pop 889 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 890 891 // Copy the result values into the output registers. 892 for (unsigned i = 0; i != RVLocs.size(); ++i) { 893 CCValAssign &VA = RVLocs[i]; 894 assert(VA.isRegLoc() && "Can only return in registers!"); 895 SDOperand ValToCopy = Op.getOperand(i*2+1); 896 897 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 898 // the RET instruction and handled by the FP Stackifier. 899 if (RVLocs[i].getLocReg() == X86::ST0 || 900 RVLocs[i].getLocReg() == X86::ST1) { 901 // If this is a copy from an xmm register to ST(0), use an FPExtend to 902 // change the value to the FP stack register class. 903 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 904 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 905 RetOps.push_back(ValToCopy); 906 // Don't emit a copytoreg. 907 continue; 908 } 909 910 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 911 Flag = Chain.getValue(1); 912 } 913 914 // The x86-64 ABI for returning structs by value requires that we copy 915 // the sret argument into %rax for the return. We saved the argument into 916 // a virtual register in the entry block, so now we copy the value out 917 // and into %rax. 918 if (Subtarget->is64Bit() && 919 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 920 MachineFunction &MF = DAG.getMachineFunction(); 921 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 922 unsigned Reg = FuncInfo->getSRetReturnReg(); 923 if (!Reg) { 924 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 925 FuncInfo->setSRetReturnReg(Reg); 926 } 927 SDOperand Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 928 929 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 930 Flag = Chain.getValue(1); 931 } 932 933 RetOps[0] = Chain; // Update chain. 934 935 // Add the flag if we have it. 936 if (Flag.Val) 937 RetOps.push_back(Flag); 938 939 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 940} 941 942 943/// LowerCallResult - Lower the result values of an ISD::CALL into the 944/// appropriate copies out of appropriate physical registers. This assumes that 945/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 946/// being lowered. The returns a SDNode with the same number of values as the 947/// ISD::CALL. 948SDNode *X86TargetLowering:: 949LowerCallResult(SDOperand Chain, SDOperand InFlag, SDNode *TheCall, 950 unsigned CallingConv, SelectionDAG &DAG) { 951 952 // Assign locations to each value returned by this call. 953 SmallVector<CCValAssign, 16> RVLocs; 954 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 955 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 956 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 957 958 SmallVector<SDOperand, 8> ResultVals; 959 960 // Copy all of the result registers out of their specified physreg. 961 for (unsigned i = 0; i != RVLocs.size(); ++i) { 962 MVT CopyVT = RVLocs[i].getValVT(); 963 964 // If this is a call to a function that returns an fp value on the floating 965 // point stack, but where we prefer to use the value in xmm registers, copy 966 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 967 if (RVLocs[i].getLocReg() == X86::ST0 && 968 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 969 CopyVT = MVT::f80; 970 } 971 972 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 973 CopyVT, InFlag).getValue(1); 974 SDOperand Val = Chain.getValue(0); 975 InFlag = Chain.getValue(2); 976 977 if (CopyVT != RVLocs[i].getValVT()) { 978 // Round the F80 the right size, which also moves to the appropriate xmm 979 // register. 980 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 981 // This truncation won't change the value. 982 DAG.getIntPtrConstant(1)); 983 } 984 985 ResultVals.push_back(Val); 986 } 987 988 // Merge everything together with a MERGE_VALUES node. 989 ResultVals.push_back(Chain); 990 return DAG.getMergeValues(TheCall->getVTList(), &ResultVals[0], 991 ResultVals.size()).Val; 992} 993 994 995//===----------------------------------------------------------------------===// 996// C & StdCall & Fast Calling Convention implementation 997//===----------------------------------------------------------------------===// 998// StdCall calling convention seems to be standard for many Windows' API 999// routines and around. It differs from C calling convention just a little: 1000// callee should clean up the stack, not caller. Symbols should be also 1001// decorated in some fancy way :) It doesn't support any vector arguments. 1002// For info on fast calling convention see Fast Calling Convention (tail call) 1003// implementation LowerX86_32FastCCCallTo. 1004 1005/// AddLiveIn - This helper function adds the specified physical register to the 1006/// MachineFunction as a live in value. It also creates a corresponding virtual 1007/// register for it. 1008static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1009 const TargetRegisterClass *RC) { 1010 assert(RC->contains(PReg) && "Not the correct regclass!"); 1011 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1012 MF.getRegInfo().addLiveIn(PReg, VReg); 1013 return VReg; 1014} 1015 1016/// CallIsStructReturn - Determines whether a CALL node uses struct return 1017/// semantics. 1018static bool CallIsStructReturn(SDOperand Op) { 1019 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1020 if (!NumOps) 1021 return false; 1022 1023 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1024} 1025 1026/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1027/// return semantics. 1028static bool ArgsAreStructReturn(SDOperand Op) { 1029 unsigned NumArgs = Op.Val->getNumValues() - 1; 1030 if (!NumArgs) 1031 return false; 1032 1033 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1034} 1035 1036/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1037/// the callee to pop its own arguments. Callee pop is necessary to support tail 1038/// calls. 1039bool X86TargetLowering::IsCalleePop(SDOperand Op) { 1040 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1041 if (IsVarArg) 1042 return false; 1043 1044 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1045 default: 1046 return false; 1047 case CallingConv::X86_StdCall: 1048 return !Subtarget->is64Bit(); 1049 case CallingConv::X86_FastCall: 1050 return !Subtarget->is64Bit(); 1051 case CallingConv::Fast: 1052 return PerformTailCallOpt; 1053 } 1054} 1055 1056/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1057/// FORMAL_ARGUMENTS node. 1058CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDOperand Op) const { 1059 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1060 1061 if (Subtarget->is64Bit()) { 1062 if (Subtarget->isTargetWin64()) 1063 return CC_X86_Win64_C; 1064 else { 1065 if (CC == CallingConv::Fast && PerformTailCallOpt) 1066 return CC_X86_64_TailCall; 1067 else 1068 return CC_X86_64_C; 1069 } 1070 } 1071 1072 if (CC == CallingConv::X86_FastCall) 1073 return CC_X86_32_FastCall; 1074 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1075 return CC_X86_32_TailCall; 1076 else 1077 return CC_X86_32_C; 1078} 1079 1080/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1081/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1082NameDecorationStyle 1083X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDOperand Op) { 1084 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1085 if (CC == CallingConv::X86_FastCall) 1086 return FastCall; 1087 else if (CC == CallingConv::X86_StdCall) 1088 return StdCall; 1089 return None; 1090} 1091 1092 1093/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1094/// in a register before calling. 1095bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1096 return !IsTailCall && !Is64Bit && 1097 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1098 Subtarget->isPICStyleGOT(); 1099} 1100 1101/// CallRequiresFnAddressInReg - Check whether the call requires the function 1102/// address to be loaded in a register. 1103bool 1104X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1105 return !Is64Bit && IsTailCall && 1106 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1107 Subtarget->isPICStyleGOT(); 1108} 1109 1110/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1111/// by "Src" to address "Dst" with size and alignment information specified by 1112/// the specific parameter attribute. The copy will be passed as a byval 1113/// function parameter. 1114static SDOperand 1115CreateCopyOfByValArgument(SDOperand Src, SDOperand Dst, SDOperand Chain, 1116 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1117 SDOperand SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1118 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1119 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1120} 1121 1122SDOperand X86TargetLowering::LowerMemArgument(SDOperand Op, SelectionDAG &DAG, 1123 const CCValAssign &VA, 1124 MachineFrameInfo *MFI, 1125 unsigned CC, 1126 SDOperand Root, unsigned i) { 1127 // Create the nodes corresponding to a load from this parameter slot. 1128 ISD::ArgFlagsTy Flags = 1129 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1130 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1131 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1132 1133 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1134 // changed with more analysis. 1135 // In case of tail call optimization mark all arguments mutable. Since they 1136 // could be overwritten by lowering of arguments in case of a tail call. 1137 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1138 VA.getLocMemOffset(), isImmutable); 1139 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); 1140 if (Flags.isByVal()) 1141 return FIN; 1142 return DAG.getLoad(VA.getValVT(), Root, FIN, 1143 PseudoSourceValue::getFixedStack(FI), 0); 1144} 1145 1146SDOperand 1147X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) { 1148 MachineFunction &MF = DAG.getMachineFunction(); 1149 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1150 1151 const Function* Fn = MF.getFunction(); 1152 if (Fn->hasExternalLinkage() && 1153 Subtarget->isTargetCygMing() && 1154 Fn->getName() == "main") 1155 FuncInfo->setForceFramePointer(true); 1156 1157 // Decorate the function name. 1158 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1159 1160 MachineFrameInfo *MFI = MF.getFrameInfo(); 1161 SDOperand Root = Op.getOperand(0); 1162 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1163 unsigned CC = MF.getFunction()->getCallingConv(); 1164 bool Is64Bit = Subtarget->is64Bit(); 1165 bool IsWin64 = Subtarget->isTargetWin64(); 1166 1167 assert(!(isVarArg && CC == CallingConv::Fast) && 1168 "Var args not supported with calling convention fastcc"); 1169 1170 // Assign locations to all of the incoming arguments. 1171 SmallVector<CCValAssign, 16> ArgLocs; 1172 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1173 CCInfo.AnalyzeFormalArguments(Op.Val, CCAssignFnForNode(Op)); 1174 1175 SmallVector<SDOperand, 8> ArgValues; 1176 unsigned LastVal = ~0U; 1177 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1178 CCValAssign &VA = ArgLocs[i]; 1179 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1180 // places. 1181 assert(VA.getValNo() != LastVal && 1182 "Don't support value assigned to multiple locs yet"); 1183 LastVal = VA.getValNo(); 1184 1185 if (VA.isRegLoc()) { 1186 MVT RegVT = VA.getLocVT(); 1187 TargetRegisterClass *RC; 1188 if (RegVT == MVT::i32) 1189 RC = X86::GR32RegisterClass; 1190 else if (Is64Bit && RegVT == MVT::i64) 1191 RC = X86::GR64RegisterClass; 1192 else if (RegVT == MVT::f32) 1193 RC = X86::FR32RegisterClass; 1194 else if (RegVT == MVT::f64) 1195 RC = X86::FR64RegisterClass; 1196 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1197 RC = X86::VR128RegisterClass; 1198 else if (RegVT.isVector()) { 1199 assert(RegVT.getSizeInBits() == 64); 1200 if (!Is64Bit) 1201 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1202 else { 1203 // Darwin calling convention passes MMX values in either GPRs or 1204 // XMMs in x86-64. Other targets pass them in memory. 1205 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1206 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1207 RegVT = MVT::v2i64; 1208 } else { 1209 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1210 RegVT = MVT::i64; 1211 } 1212 } 1213 } else { 1214 assert(0 && "Unknown argument type!"); 1215 } 1216 1217 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1218 SDOperand ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1219 1220 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1221 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1222 // right size. 1223 if (VA.getLocInfo() == CCValAssign::SExt) 1224 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1225 DAG.getValueType(VA.getValVT())); 1226 else if (VA.getLocInfo() == CCValAssign::ZExt) 1227 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1228 DAG.getValueType(VA.getValVT())); 1229 1230 if (VA.getLocInfo() != CCValAssign::Full) 1231 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1232 1233 // Handle MMX values passed in GPRs. 1234 if (Is64Bit && RegVT != VA.getLocVT()) { 1235 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1236 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1237 else if (RC == X86::VR128RegisterClass) { 1238 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1239 DAG.getConstant(0, MVT::i64)); 1240 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1241 } 1242 } 1243 1244 ArgValues.push_back(ArgValue); 1245 } else { 1246 assert(VA.isMemLoc()); 1247 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1248 } 1249 } 1250 1251 // The x86-64 ABI for returning structs by value requires that we copy 1252 // the sret argument into %rax for the return. Save the argument into 1253 // a virtual register so that we can access it from the return points. 1254 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1255 MachineFunction &MF = DAG.getMachineFunction(); 1256 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1257 unsigned Reg = FuncInfo->getSRetReturnReg(); 1258 if (!Reg) { 1259 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1260 FuncInfo->setSRetReturnReg(Reg); 1261 } 1262 SDOperand Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1263 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1264 } 1265 1266 unsigned StackSize = CCInfo.getNextStackOffset(); 1267 // align stack specially for tail calls 1268 if (CC == CallingConv::Fast) 1269 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1270 1271 // If the function takes variable number of arguments, make a frame index for 1272 // the start of the first vararg value... for expansion of llvm.va_start. 1273 if (isVarArg) { 1274 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1275 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1276 } 1277 if (Is64Bit) { 1278 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1279 1280 // FIXME: We should really autogenerate these arrays 1281 static const unsigned GPR64ArgRegsWin64[] = { 1282 X86::RCX, X86::RDX, X86::R8, X86::R9 1283 }; 1284 static const unsigned XMMArgRegsWin64[] = { 1285 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1286 }; 1287 static const unsigned GPR64ArgRegs64Bit[] = { 1288 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1289 }; 1290 static const unsigned XMMArgRegs64Bit[] = { 1291 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1292 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1293 }; 1294 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1295 1296 if (IsWin64) { 1297 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1298 GPR64ArgRegs = GPR64ArgRegsWin64; 1299 XMMArgRegs = XMMArgRegsWin64; 1300 } else { 1301 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1302 GPR64ArgRegs = GPR64ArgRegs64Bit; 1303 XMMArgRegs = XMMArgRegs64Bit; 1304 } 1305 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1306 TotalNumIntRegs); 1307 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1308 TotalNumXMMRegs); 1309 1310 // For X86-64, if there are vararg parameters that are passed via 1311 // registers, then we must store them to their spots on the stack so they 1312 // may be loaded by deferencing the result of va_next. 1313 VarArgsGPOffset = NumIntRegs * 8; 1314 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1315 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1316 TotalNumXMMRegs * 16, 16); 1317 1318 // Store the integer parameter registers. 1319 SmallVector<SDOperand, 8> MemOps; 1320 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1321 SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1322 DAG.getIntPtrConstant(VarArgsGPOffset)); 1323 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1324 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1325 X86::GR64RegisterClass); 1326 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1327 SDOperand Store = 1328 DAG.getStore(Val.getValue(1), Val, FIN, 1329 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1330 MemOps.push_back(Store); 1331 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1332 DAG.getIntPtrConstant(8)); 1333 } 1334 1335 // Now store the XMM (fp + vector) parameter registers. 1336 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1337 DAG.getIntPtrConstant(VarArgsFPOffset)); 1338 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1339 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1340 X86::VR128RegisterClass); 1341 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1342 SDOperand Store = 1343 DAG.getStore(Val.getValue(1), Val, FIN, 1344 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1345 MemOps.push_back(Store); 1346 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1347 DAG.getIntPtrConstant(16)); 1348 } 1349 if (!MemOps.empty()) 1350 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1351 &MemOps[0], MemOps.size()); 1352 } 1353 } 1354 1355 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1356 // arguments and the arguments after the retaddr has been pushed are 1357 // aligned. 1358 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1359 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1360 (StackSize & 7) == 0) 1361 StackSize += 4; 1362 1363 ArgValues.push_back(Root); 1364 1365 // Some CCs need callee pop. 1366 if (IsCalleePop(Op)) { 1367 BytesToPopOnReturn = StackSize; // Callee pops everything. 1368 BytesCallerReserves = 0; 1369 } else { 1370 BytesToPopOnReturn = 0; // Callee pops nothing. 1371 // If this is an sret function, the return should pop the hidden pointer. 1372 if (!Is64Bit && ArgsAreStructReturn(Op)) 1373 BytesToPopOnReturn = 4; 1374 BytesCallerReserves = StackSize; 1375 } 1376 1377 if (!Is64Bit) { 1378 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1379 if (CC == CallingConv::X86_FastCall) 1380 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1381 } 1382 1383 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1384 1385 // Return the new list of results. 1386 return DAG.getMergeValues(Op.Val->getVTList(), &ArgValues[0], 1387 ArgValues.size()).getValue(Op.ResNo); 1388} 1389 1390SDOperand 1391X86TargetLowering::LowerMemOpCallTo(SDOperand Op, SelectionDAG &DAG, 1392 const SDOperand &StackPtr, 1393 const CCValAssign &VA, 1394 SDOperand Chain, 1395 SDOperand Arg) { 1396 unsigned LocMemOffset = VA.getLocMemOffset(); 1397 SDOperand PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1398 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1399 ISD::ArgFlagsTy Flags = 1400 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1401 if (Flags.isByVal()) { 1402 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1403 } 1404 return DAG.getStore(Chain, Arg, PtrOff, 1405 PseudoSourceValue::getStack(), LocMemOffset); 1406} 1407 1408/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1409/// optimization is performed and it is required. 1410SDOperand 1411X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1412 SDOperand &OutRetAddr, 1413 SDOperand Chain, 1414 bool IsTailCall, 1415 bool Is64Bit, 1416 int FPDiff) { 1417 if (!IsTailCall || FPDiff==0) return Chain; 1418 1419 // Adjust the Return address stack slot. 1420 MVT VT = getPointerTy(); 1421 OutRetAddr = getReturnAddressFrameIndex(DAG); 1422 // Load the "old" Return address. 1423 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1424 return SDOperand(OutRetAddr.Val, 1); 1425} 1426 1427/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1428/// optimization is performed and it is required (FPDiff!=0). 1429static SDOperand 1430EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1431 SDOperand Chain, SDOperand RetAddrFrIdx, 1432 bool Is64Bit, int FPDiff) { 1433 // Store the return address to the appropriate stack slot. 1434 if (!FPDiff) return Chain; 1435 // Calculate the new stack slot for the return address. 1436 int SlotSize = Is64Bit ? 8 : 4; 1437 int NewReturnAddrFI = 1438 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1439 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1440 SDOperand NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1441 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1442 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 1443 return Chain; 1444} 1445 1446SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) { 1447 MachineFunction &MF = DAG.getMachineFunction(); 1448 SDOperand Chain = Op.getOperand(0); 1449 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1450 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1451 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1452 && CC == CallingConv::Fast && PerformTailCallOpt; 1453 SDOperand Callee = Op.getOperand(4); 1454 bool Is64Bit = Subtarget->is64Bit(); 1455 bool IsStructRet = CallIsStructReturn(Op); 1456 1457 assert(!(isVarArg && CC == CallingConv::Fast) && 1458 "Var args not supported with calling convention fastcc"); 1459 1460 // Analyze operands of the call, assigning locations to each operand. 1461 SmallVector<CCValAssign, 16> ArgLocs; 1462 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1463 CCInfo.AnalyzeCallOperands(Op.Val, CCAssignFnForNode(Op)); 1464 1465 // Get a count of how many bytes are to be pushed on the stack. 1466 unsigned NumBytes = CCInfo.getNextStackOffset(); 1467 if (CC == CallingConv::Fast) 1468 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1469 1470 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1471 // arguments and the arguments after the retaddr has been pushed are aligned. 1472 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1473 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1474 (NumBytes & 7) == 0) 1475 NumBytes += 4; 1476 1477 int FPDiff = 0; 1478 if (IsTailCall) { 1479 // Lower arguments at fp - stackoffset + fpdiff. 1480 unsigned NumBytesCallerPushed = 1481 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1482 FPDiff = NumBytesCallerPushed - NumBytes; 1483 1484 // Set the delta of movement of the returnaddr stackslot. 1485 // But only set if delta is greater than previous delta. 1486 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1487 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1488 } 1489 1490 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1491 1492 SDOperand RetAddrFrIdx; 1493 // Load return adress for tail calls. 1494 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1495 FPDiff); 1496 1497 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass; 1498 SmallVector<SDOperand, 8> MemOpChains; 1499 SDOperand StackPtr; 1500 1501 // Walk the register/memloc assignments, inserting copies/loads. In the case 1502 // of tail call optimization arguments are handle later. 1503 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1504 CCValAssign &VA = ArgLocs[i]; 1505 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1506 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1507 getArgFlags().isByVal(); 1508 1509 // Promote the value if needed. 1510 switch (VA.getLocInfo()) { 1511 default: assert(0 && "Unknown loc info!"); 1512 case CCValAssign::Full: break; 1513 case CCValAssign::SExt: 1514 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1515 break; 1516 case CCValAssign::ZExt: 1517 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1518 break; 1519 case CCValAssign::AExt: 1520 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1521 break; 1522 } 1523 1524 if (VA.isRegLoc()) { 1525 if (Is64Bit) { 1526 MVT RegVT = VA.getLocVT(); 1527 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1528 switch (VA.getLocReg()) { 1529 default: 1530 break; 1531 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1532 case X86::R8: { 1533 // Special case: passing MMX values in GPR registers. 1534 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1535 break; 1536 } 1537 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1538 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1539 // Special case: passing MMX values in XMM registers. 1540 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1541 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1542 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1543 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1544 getMOVLMask(2, DAG)); 1545 break; 1546 } 1547 } 1548 } 1549 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1550 } else { 1551 if (!IsTailCall || (IsTailCall && isByVal)) { 1552 assert(VA.isMemLoc()); 1553 if (StackPtr.Val == 0) 1554 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1555 1556 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1557 Arg)); 1558 } 1559 } 1560 } 1561 1562 if (!MemOpChains.empty()) 1563 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1564 &MemOpChains[0], MemOpChains.size()); 1565 1566 // Build a sequence of copy-to-reg nodes chained together with token chain 1567 // and flag operands which copy the outgoing args into registers. 1568 SDOperand InFlag; 1569 // Tail call byval lowering might overwrite argument registers so in case of 1570 // tail call optimization the copies to registers are lowered later. 1571 if (!IsTailCall) 1572 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1573 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1574 InFlag); 1575 InFlag = Chain.getValue(1); 1576 } 1577 1578 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1579 // GOT pointer. 1580 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1581 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1582 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1583 InFlag); 1584 InFlag = Chain.getValue(1); 1585 } 1586 // If we are tail calling and generating PIC/GOT style code load the address 1587 // of the callee into ecx. The value in ecx is used as target of the tail 1588 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1589 // calls on PIC/GOT architectures. Normally we would just put the address of 1590 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1591 // restored (since ebx is callee saved) before jumping to the target@PLT. 1592 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1593 // Note: The actual moving to ecx is done further down. 1594 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1595 if (G && !G->getGlobal()->hasHiddenVisibility() && 1596 !G->getGlobal()->hasProtectedVisibility()) 1597 Callee = LowerGlobalAddress(Callee, DAG); 1598 else if (isa<ExternalSymbolSDNode>(Callee)) 1599 Callee = LowerExternalSymbol(Callee,DAG); 1600 } 1601 1602 if (Is64Bit && isVarArg) { 1603 // From AMD64 ABI document: 1604 // For calls that may call functions that use varargs or stdargs 1605 // (prototype-less calls or calls to functions containing ellipsis (...) in 1606 // the declaration) %al is used as hidden argument to specify the number 1607 // of SSE registers used. The contents of %al do not need to match exactly 1608 // the number of registers, but must be an ubound on the number of SSE 1609 // registers used and is in the range 0 - 8 inclusive. 1610 1611 // FIXME: Verify this on Win64 1612 // Count the number of XMM registers allocated. 1613 static const unsigned XMMArgRegs[] = { 1614 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1615 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1616 }; 1617 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1618 1619 Chain = DAG.getCopyToReg(Chain, X86::AL, 1620 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1621 InFlag = Chain.getValue(1); 1622 } 1623 1624 1625 // For tail calls lower the arguments to the 'real' stack slot. 1626 if (IsTailCall) { 1627 SmallVector<SDOperand, 8> MemOpChains2; 1628 SDOperand FIN; 1629 int FI = 0; 1630 // Do not flag preceeding copytoreg stuff together with the following stuff. 1631 InFlag = SDOperand(); 1632 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1633 CCValAssign &VA = ArgLocs[i]; 1634 if (!VA.isRegLoc()) { 1635 assert(VA.isMemLoc()); 1636 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1637 SDOperand FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1638 ISD::ArgFlagsTy Flags = 1639 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1640 // Create frame index. 1641 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1642 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1643 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1644 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1645 1646 if (Flags.isByVal()) { 1647 // Copy relative to framepointer. 1648 SDOperand Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1649 if (StackPtr.Val == 0) 1650 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1651 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1652 1653 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1654 Flags, DAG)); 1655 } else { 1656 // Store relative to framepointer. 1657 MemOpChains2.push_back( 1658 DAG.getStore(Chain, Arg, FIN, 1659 PseudoSourceValue::getFixedStack(FI), 0)); 1660 } 1661 } 1662 } 1663 1664 if (!MemOpChains2.empty()) 1665 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1666 &MemOpChains2[0], MemOpChains2.size()); 1667 1668 // Copy arguments to their registers. 1669 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1670 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1671 InFlag); 1672 InFlag = Chain.getValue(1); 1673 } 1674 InFlag =SDOperand(); 1675 1676 // Store the return address to the appropriate stack slot. 1677 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1678 FPDiff); 1679 } 1680 1681 // If the callee is a GlobalAddress node (quite common, every direct call is) 1682 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1683 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1684 // We should use extra load for direct calls to dllimported functions in 1685 // non-JIT mode. 1686 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1687 getTargetMachine(), true)) 1688 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1689 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1690 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1691 } else if (IsTailCall) { 1692 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1693 1694 Chain = DAG.getCopyToReg(Chain, 1695 DAG.getRegister(Opc, getPointerTy()), 1696 Callee,InFlag); 1697 Callee = DAG.getRegister(Opc, getPointerTy()); 1698 // Add register as live out. 1699 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1700 } 1701 1702 // Returns a chain & a flag for retval copy to use. 1703 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1704 SmallVector<SDOperand, 8> Ops; 1705 1706 if (IsTailCall) { 1707 Ops.push_back(Chain); 1708 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1709 Ops.push_back(DAG.getIntPtrConstant(0)); 1710 if (InFlag.Val) 1711 Ops.push_back(InFlag); 1712 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1713 InFlag = Chain.getValue(1); 1714 1715 // Returns a chain & a flag for retval copy to use. 1716 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1717 Ops.clear(); 1718 } 1719 1720 Ops.push_back(Chain); 1721 Ops.push_back(Callee); 1722 1723 if (IsTailCall) 1724 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1725 1726 // Add argument registers to the end of the list so that they are known live 1727 // into the call. 1728 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1729 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1730 RegsToPass[i].second.getValueType())); 1731 1732 // Add an implicit use GOT pointer in EBX. 1733 if (!IsTailCall && !Is64Bit && 1734 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1735 Subtarget->isPICStyleGOT()) 1736 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1737 1738 // Add an implicit use of AL for x86 vararg functions. 1739 if (Is64Bit && isVarArg) 1740 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1741 1742 if (InFlag.Val) 1743 Ops.push_back(InFlag); 1744 1745 if (IsTailCall) { 1746 assert(InFlag.Val && 1747 "Flag must be set. Depend on flag being set in LowerRET"); 1748 Chain = DAG.getNode(X86ISD::TAILCALL, 1749 Op.Val->getVTList(), &Ops[0], Ops.size()); 1750 1751 return SDOperand(Chain.Val, Op.ResNo); 1752 } 1753 1754 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1755 InFlag = Chain.getValue(1); 1756 1757 // Create the CALLSEQ_END node. 1758 unsigned NumBytesForCalleeToPush; 1759 if (IsCalleePop(Op)) 1760 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1761 else if (!Is64Bit && IsStructRet) 1762 // If this is is a call to a struct-return function, the callee 1763 // pops the hidden struct pointer, so we have to push it back. 1764 // This is common for Darwin/X86, Linux & Mingw32 targets. 1765 NumBytesForCalleeToPush = 4; 1766 else 1767 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1768 1769 // Returns a flag for retval copy to use. 1770 Chain = DAG.getCALLSEQ_END(Chain, 1771 DAG.getIntPtrConstant(NumBytes), 1772 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1773 InFlag); 1774 InFlag = Chain.getValue(1); 1775 1776 // Handle result values, copying them out of physregs into vregs that we 1777 // return. 1778 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo); 1779} 1780 1781 1782//===----------------------------------------------------------------------===// 1783// Fast Calling Convention (tail call) implementation 1784//===----------------------------------------------------------------------===// 1785 1786// Like std call, callee cleans arguments, convention except that ECX is 1787// reserved for storing the tail called function address. Only 2 registers are 1788// free for argument passing (inreg). Tail call optimization is performed 1789// provided: 1790// * tailcallopt is enabled 1791// * caller/callee are fastcc 1792// On X86_64 architecture with GOT-style position independent code only local 1793// (within module) calls are supported at the moment. 1794// To keep the stack aligned according to platform abi the function 1795// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1796// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1797// If a tail called function callee has more arguments than the caller the 1798// caller needs to make sure that there is room to move the RETADDR to. This is 1799// achieved by reserving an area the size of the argument delta right after the 1800// original REtADDR, but before the saved framepointer or the spilled registers 1801// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1802// stack layout: 1803// arg1 1804// arg2 1805// RETADDR 1806// [ new RETADDR 1807// move area ] 1808// (possible EBP) 1809// ESI 1810// EDI 1811// local1 .. 1812 1813/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1814/// for a 16 byte align requirement. 1815unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1816 SelectionDAG& DAG) { 1817 if (PerformTailCallOpt) { 1818 MachineFunction &MF = DAG.getMachineFunction(); 1819 const TargetMachine &TM = MF.getTarget(); 1820 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1821 unsigned StackAlignment = TFI.getStackAlignment(); 1822 uint64_t AlignMask = StackAlignment - 1; 1823 int64_t Offset = StackSize; 1824 unsigned SlotSize = Subtarget->is64Bit() ? 8 : 4; 1825 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1826 // Number smaller than 12 so just add the difference. 1827 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1828 } else { 1829 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1830 Offset = ((~AlignMask) & Offset) + StackAlignment + 1831 (StackAlignment-SlotSize); 1832 } 1833 StackSize = Offset; 1834 } 1835 return StackSize; 1836} 1837 1838/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1839/// following the call is a return. A function is eligible if caller/callee 1840/// calling conventions match, currently only fastcc supports tail calls, and 1841/// the function CALL is immediatly followed by a RET. 1842bool X86TargetLowering::IsEligibleForTailCallOptimization(SDOperand Call, 1843 SDOperand Ret, 1844 SelectionDAG& DAG) const { 1845 if (!PerformTailCallOpt) 1846 return false; 1847 1848 if (CheckTailCallReturnConstraints(Call, Ret)) { 1849 MachineFunction &MF = DAG.getMachineFunction(); 1850 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1851 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1852 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1853 SDOperand Callee = Call.getOperand(4); 1854 // On x86/32Bit PIC/GOT tail calls are supported. 1855 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1856 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1857 return true; 1858 1859 // Can only do local tail calls (in same module, hidden or protected) on 1860 // x86_64 PIC/GOT at the moment. 1861 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1862 return G->getGlobal()->hasHiddenVisibility() 1863 || G->getGlobal()->hasProtectedVisibility(); 1864 } 1865 } 1866 1867 return false; 1868} 1869 1870//===----------------------------------------------------------------------===// 1871// Other Lowering Hooks 1872//===----------------------------------------------------------------------===// 1873 1874 1875SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1876 MachineFunction &MF = DAG.getMachineFunction(); 1877 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1878 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1879 1880 if (ReturnAddrIndex == 0) { 1881 // Set up a frame object for the return address. 1882 if (Subtarget->is64Bit()) 1883 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); 1884 else 1885 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1886 1887 FuncInfo->setRAIndex(ReturnAddrIndex); 1888 } 1889 1890 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1891} 1892 1893 1894 1895/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1896/// specific condition code. It returns a false if it cannot do a direct 1897/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1898/// needed. 1899static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1900 unsigned &X86CC, SDOperand &LHS, SDOperand &RHS, 1901 SelectionDAG &DAG) { 1902 X86CC = X86::COND_INVALID; 1903 if (!isFP) { 1904 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1905 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1906 // X > -1 -> X == 0, jump !sign. 1907 RHS = DAG.getConstant(0, RHS.getValueType()); 1908 X86CC = X86::COND_NS; 1909 return true; 1910 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1911 // X < 0 -> X == 0, jump on sign. 1912 X86CC = X86::COND_S; 1913 return true; 1914 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1915 // X < 1 -> X <= 0 1916 RHS = DAG.getConstant(0, RHS.getValueType()); 1917 X86CC = X86::COND_LE; 1918 return true; 1919 } 1920 } 1921 1922 switch (SetCCOpcode) { 1923 default: break; 1924 case ISD::SETEQ: X86CC = X86::COND_E; break; 1925 case ISD::SETGT: X86CC = X86::COND_G; break; 1926 case ISD::SETGE: X86CC = X86::COND_GE; break; 1927 case ISD::SETLT: X86CC = X86::COND_L; break; 1928 case ISD::SETLE: X86CC = X86::COND_LE; break; 1929 case ISD::SETNE: X86CC = X86::COND_NE; break; 1930 case ISD::SETULT: X86CC = X86::COND_B; break; 1931 case ISD::SETUGT: X86CC = X86::COND_A; break; 1932 case ISD::SETULE: X86CC = X86::COND_BE; break; 1933 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1934 } 1935 } else { 1936 // On a floating point condition, the flags are set as follows: 1937 // ZF PF CF op 1938 // 0 | 0 | 0 | X > Y 1939 // 0 | 0 | 1 | X < Y 1940 // 1 | 0 | 0 | X == Y 1941 // 1 | 1 | 1 | unordered 1942 bool Flip = false; 1943 switch (SetCCOpcode) { 1944 default: break; 1945 case ISD::SETUEQ: 1946 case ISD::SETEQ: X86CC = X86::COND_E; break; 1947 case ISD::SETOLT: Flip = true; // Fallthrough 1948 case ISD::SETOGT: 1949 case ISD::SETGT: X86CC = X86::COND_A; break; 1950 case ISD::SETOLE: Flip = true; // Fallthrough 1951 case ISD::SETOGE: 1952 case ISD::SETGE: X86CC = X86::COND_AE; break; 1953 case ISD::SETUGT: Flip = true; // Fallthrough 1954 case ISD::SETULT: 1955 case ISD::SETLT: X86CC = X86::COND_B; break; 1956 case ISD::SETUGE: Flip = true; // Fallthrough 1957 case ISD::SETULE: 1958 case ISD::SETLE: X86CC = X86::COND_BE; break; 1959 case ISD::SETONE: 1960 case ISD::SETNE: X86CC = X86::COND_NE; break; 1961 case ISD::SETUO: X86CC = X86::COND_P; break; 1962 case ISD::SETO: X86CC = X86::COND_NP; break; 1963 } 1964 if (Flip) 1965 std::swap(LHS, RHS); 1966 } 1967 1968 return X86CC != X86::COND_INVALID; 1969} 1970 1971/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1972/// code. Current x86 isa includes the following FP cmov instructions: 1973/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1974static bool hasFPCMov(unsigned X86CC) { 1975 switch (X86CC) { 1976 default: 1977 return false; 1978 case X86::COND_B: 1979 case X86::COND_BE: 1980 case X86::COND_E: 1981 case X86::COND_P: 1982 case X86::COND_A: 1983 case X86::COND_AE: 1984 case X86::COND_NE: 1985 case X86::COND_NP: 1986 return true; 1987 } 1988} 1989 1990/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 1991/// true if Op is undef or if its value falls within the specified range (L, H]. 1992static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) { 1993 if (Op.getOpcode() == ISD::UNDEF) 1994 return true; 1995 1996 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 1997 return (Val >= Low && Val < Hi); 1998} 1999 2000/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 2001/// true if Op is undef or if its value equal to the specified value. 2002static bool isUndefOrEqual(SDOperand Op, unsigned Val) { 2003 if (Op.getOpcode() == ISD::UNDEF) 2004 return true; 2005 return cast<ConstantSDNode>(Op)->getValue() == Val; 2006} 2007 2008/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2009/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2010bool X86::isPSHUFDMask(SDNode *N) { 2011 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2012 2013 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2014 return false; 2015 2016 // Check if the value doesn't reference the second vector. 2017 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2018 SDOperand Arg = N->getOperand(i); 2019 if (Arg.getOpcode() == ISD::UNDEF) continue; 2020 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2021 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2022 return false; 2023 } 2024 2025 return true; 2026} 2027 2028/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2029/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2030bool X86::isPSHUFHWMask(SDNode *N) { 2031 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2032 2033 if (N->getNumOperands() != 8) 2034 return false; 2035 2036 // Lower quadword copied in order. 2037 for (unsigned i = 0; i != 4; ++i) { 2038 SDOperand Arg = N->getOperand(i); 2039 if (Arg.getOpcode() == ISD::UNDEF) continue; 2040 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2041 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2042 return false; 2043 } 2044 2045 // Upper quadword shuffled. 2046 for (unsigned i = 4; i != 8; ++i) { 2047 SDOperand Arg = N->getOperand(i); 2048 if (Arg.getOpcode() == ISD::UNDEF) continue; 2049 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2050 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2051 if (Val < 4 || Val > 7) 2052 return false; 2053 } 2054 2055 return true; 2056} 2057 2058/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2059/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2060bool X86::isPSHUFLWMask(SDNode *N) { 2061 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2062 2063 if (N->getNumOperands() != 8) 2064 return false; 2065 2066 // Upper quadword copied in order. 2067 for (unsigned i = 4; i != 8; ++i) 2068 if (!isUndefOrEqual(N->getOperand(i), i)) 2069 return false; 2070 2071 // Lower quadword shuffled. 2072 for (unsigned i = 0; i != 4; ++i) 2073 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2074 return false; 2075 2076 return true; 2077} 2078 2079/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2080/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2081static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2082 if (NumElems != 2 && NumElems != 4) return false; 2083 2084 unsigned Half = NumElems / 2; 2085 for (unsigned i = 0; i < Half; ++i) 2086 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2087 return false; 2088 for (unsigned i = Half; i < NumElems; ++i) 2089 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2090 return false; 2091 2092 return true; 2093} 2094 2095bool X86::isSHUFPMask(SDNode *N) { 2096 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2097 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2098} 2099 2100/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2101/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2102/// half elements to come from vector 1 (which would equal the dest.) and 2103/// the upper half to come from vector 2. 2104static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2105 if (NumOps != 2 && NumOps != 4) return false; 2106 2107 unsigned Half = NumOps / 2; 2108 for (unsigned i = 0; i < Half; ++i) 2109 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2110 return false; 2111 for (unsigned i = Half; i < NumOps; ++i) 2112 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2113 return false; 2114 return true; 2115} 2116 2117static bool isCommutedSHUFP(SDNode *N) { 2118 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2119 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2120} 2121 2122/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2123/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2124bool X86::isMOVHLPSMask(SDNode *N) { 2125 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2126 2127 if (N->getNumOperands() != 4) 2128 return false; 2129 2130 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2131 return isUndefOrEqual(N->getOperand(0), 6) && 2132 isUndefOrEqual(N->getOperand(1), 7) && 2133 isUndefOrEqual(N->getOperand(2), 2) && 2134 isUndefOrEqual(N->getOperand(3), 3); 2135} 2136 2137/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2138/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2139/// <2, 3, 2, 3> 2140bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2141 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2142 2143 if (N->getNumOperands() != 4) 2144 return false; 2145 2146 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2147 return isUndefOrEqual(N->getOperand(0), 2) && 2148 isUndefOrEqual(N->getOperand(1), 3) && 2149 isUndefOrEqual(N->getOperand(2), 2) && 2150 isUndefOrEqual(N->getOperand(3), 3); 2151} 2152 2153/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2154/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2155bool X86::isMOVLPMask(SDNode *N) { 2156 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2157 2158 unsigned NumElems = N->getNumOperands(); 2159 if (NumElems != 2 && NumElems != 4) 2160 return false; 2161 2162 for (unsigned i = 0; i < NumElems/2; ++i) 2163 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2164 return false; 2165 2166 for (unsigned i = NumElems/2; i < NumElems; ++i) 2167 if (!isUndefOrEqual(N->getOperand(i), i)) 2168 return false; 2169 2170 return true; 2171} 2172 2173/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2174/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2175/// and MOVLHPS. 2176bool X86::isMOVHPMask(SDNode *N) { 2177 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2178 2179 unsigned NumElems = N->getNumOperands(); 2180 if (NumElems != 2 && NumElems != 4) 2181 return false; 2182 2183 for (unsigned i = 0; i < NumElems/2; ++i) 2184 if (!isUndefOrEqual(N->getOperand(i), i)) 2185 return false; 2186 2187 for (unsigned i = 0; i < NumElems/2; ++i) { 2188 SDOperand Arg = N->getOperand(i + NumElems/2); 2189 if (!isUndefOrEqual(Arg, i + NumElems)) 2190 return false; 2191 } 2192 2193 return true; 2194} 2195 2196/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2197/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2198bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2199 bool V2IsSplat = false) { 2200 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2201 return false; 2202 2203 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2204 SDOperand BitI = Elts[i]; 2205 SDOperand BitI1 = Elts[i+1]; 2206 if (!isUndefOrEqual(BitI, j)) 2207 return false; 2208 if (V2IsSplat) { 2209 if (isUndefOrEqual(BitI1, NumElts)) 2210 return false; 2211 } else { 2212 if (!isUndefOrEqual(BitI1, j + NumElts)) 2213 return false; 2214 } 2215 } 2216 2217 return true; 2218} 2219 2220bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2221 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2222 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2223} 2224 2225/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2226/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2227bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2228 bool V2IsSplat = false) { 2229 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2230 return false; 2231 2232 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2233 SDOperand BitI = Elts[i]; 2234 SDOperand BitI1 = Elts[i+1]; 2235 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2236 return false; 2237 if (V2IsSplat) { 2238 if (isUndefOrEqual(BitI1, NumElts)) 2239 return false; 2240 } else { 2241 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2242 return false; 2243 } 2244 } 2245 2246 return true; 2247} 2248 2249bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2250 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2251 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2252} 2253 2254/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2255/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2256/// <0, 0, 1, 1> 2257bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2258 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2259 2260 unsigned NumElems = N->getNumOperands(); 2261 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2262 return false; 2263 2264 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2265 SDOperand BitI = N->getOperand(i); 2266 SDOperand BitI1 = N->getOperand(i+1); 2267 2268 if (!isUndefOrEqual(BitI, j)) 2269 return false; 2270 if (!isUndefOrEqual(BitI1, j)) 2271 return false; 2272 } 2273 2274 return true; 2275} 2276 2277/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2278/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2279/// <2, 2, 3, 3> 2280bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2281 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2282 2283 unsigned NumElems = N->getNumOperands(); 2284 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2285 return false; 2286 2287 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2288 SDOperand BitI = N->getOperand(i); 2289 SDOperand BitI1 = N->getOperand(i + 1); 2290 2291 if (!isUndefOrEqual(BitI, j)) 2292 return false; 2293 if (!isUndefOrEqual(BitI1, j)) 2294 return false; 2295 } 2296 2297 return true; 2298} 2299 2300/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2301/// specifies a shuffle of elements that is suitable for input to MOVSS, 2302/// MOVSD, and MOVD, i.e. setting the lowest element. 2303static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2304 if (NumElts != 2 && NumElts != 4) 2305 return false; 2306 2307 if (!isUndefOrEqual(Elts[0], NumElts)) 2308 return false; 2309 2310 for (unsigned i = 1; i < NumElts; ++i) { 2311 if (!isUndefOrEqual(Elts[i], i)) 2312 return false; 2313 } 2314 2315 return true; 2316} 2317 2318bool X86::isMOVLMask(SDNode *N) { 2319 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2320 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2321} 2322 2323/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2324/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2325/// element of vector 2 and the other elements to come from vector 1 in order. 2326static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2327 bool V2IsSplat = false, 2328 bool V2IsUndef = false) { 2329 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2330 return false; 2331 2332 if (!isUndefOrEqual(Ops[0], 0)) 2333 return false; 2334 2335 for (unsigned i = 1; i < NumOps; ++i) { 2336 SDOperand Arg = Ops[i]; 2337 if (!(isUndefOrEqual(Arg, i+NumOps) || 2338 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2339 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2340 return false; 2341 } 2342 2343 return true; 2344} 2345 2346static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2347 bool V2IsUndef = false) { 2348 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2349 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2350 V2IsSplat, V2IsUndef); 2351} 2352 2353/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2354/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2355bool X86::isMOVSHDUPMask(SDNode *N) { 2356 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2357 2358 if (N->getNumOperands() != 4) 2359 return false; 2360 2361 // Expect 1, 1, 3, 3 2362 for (unsigned i = 0; i < 2; ++i) { 2363 SDOperand Arg = N->getOperand(i); 2364 if (Arg.getOpcode() == ISD::UNDEF) continue; 2365 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2366 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2367 if (Val != 1) return false; 2368 } 2369 2370 bool HasHi = false; 2371 for (unsigned i = 2; i < 4; ++i) { 2372 SDOperand Arg = N->getOperand(i); 2373 if (Arg.getOpcode() == ISD::UNDEF) continue; 2374 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2375 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2376 if (Val != 3) return false; 2377 HasHi = true; 2378 } 2379 2380 // Don't use movshdup if it can be done with a shufps. 2381 return HasHi; 2382} 2383 2384/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2385/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2386bool X86::isMOVSLDUPMask(SDNode *N) { 2387 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2388 2389 if (N->getNumOperands() != 4) 2390 return false; 2391 2392 // Expect 0, 0, 2, 2 2393 for (unsigned i = 0; i < 2; ++i) { 2394 SDOperand Arg = N->getOperand(i); 2395 if (Arg.getOpcode() == ISD::UNDEF) continue; 2396 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2397 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2398 if (Val != 0) return false; 2399 } 2400 2401 bool HasHi = false; 2402 for (unsigned i = 2; i < 4; ++i) { 2403 SDOperand Arg = N->getOperand(i); 2404 if (Arg.getOpcode() == ISD::UNDEF) continue; 2405 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2406 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2407 if (Val != 2) return false; 2408 HasHi = true; 2409 } 2410 2411 // Don't use movshdup if it can be done with a shufps. 2412 return HasHi; 2413} 2414 2415/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2416/// specifies a identity operation on the LHS or RHS. 2417static bool isIdentityMask(SDNode *N, bool RHS = false) { 2418 unsigned NumElems = N->getNumOperands(); 2419 for (unsigned i = 0; i < NumElems; ++i) 2420 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2421 return false; 2422 return true; 2423} 2424 2425/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2426/// a splat of a single element. 2427static bool isSplatMask(SDNode *N) { 2428 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2429 2430 // This is a splat operation if each element of the permute is the same, and 2431 // if the value doesn't reference the second vector. 2432 unsigned NumElems = N->getNumOperands(); 2433 SDOperand ElementBase; 2434 unsigned i = 0; 2435 for (; i != NumElems; ++i) { 2436 SDOperand Elt = N->getOperand(i); 2437 if (isa<ConstantSDNode>(Elt)) { 2438 ElementBase = Elt; 2439 break; 2440 } 2441 } 2442 2443 if (!ElementBase.Val) 2444 return false; 2445 2446 for (; i != NumElems; ++i) { 2447 SDOperand Arg = N->getOperand(i); 2448 if (Arg.getOpcode() == ISD::UNDEF) continue; 2449 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2450 if (Arg != ElementBase) return false; 2451 } 2452 2453 // Make sure it is a splat of the first vector operand. 2454 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2455} 2456 2457/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2458/// a splat of a single element and it's a 2 or 4 element mask. 2459bool X86::isSplatMask(SDNode *N) { 2460 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2461 2462 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2463 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2464 return false; 2465 return ::isSplatMask(N); 2466} 2467 2468/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2469/// specifies a splat of zero element. 2470bool X86::isSplatLoMask(SDNode *N) { 2471 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2472 2473 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2474 if (!isUndefOrEqual(N->getOperand(i), 0)) 2475 return false; 2476 return true; 2477} 2478 2479/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2480/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2481/// instructions. 2482unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2483 unsigned NumOperands = N->getNumOperands(); 2484 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2485 unsigned Mask = 0; 2486 for (unsigned i = 0; i < NumOperands; ++i) { 2487 unsigned Val = 0; 2488 SDOperand Arg = N->getOperand(NumOperands-i-1); 2489 if (Arg.getOpcode() != ISD::UNDEF) 2490 Val = cast<ConstantSDNode>(Arg)->getValue(); 2491 if (Val >= NumOperands) Val -= NumOperands; 2492 Mask |= Val; 2493 if (i != NumOperands - 1) 2494 Mask <<= Shift; 2495 } 2496 2497 return Mask; 2498} 2499 2500/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2501/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2502/// instructions. 2503unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2504 unsigned Mask = 0; 2505 // 8 nodes, but we only care about the last 4. 2506 for (unsigned i = 7; i >= 4; --i) { 2507 unsigned Val = 0; 2508 SDOperand Arg = N->getOperand(i); 2509 if (Arg.getOpcode() != ISD::UNDEF) 2510 Val = cast<ConstantSDNode>(Arg)->getValue(); 2511 Mask |= (Val - 4); 2512 if (i != 4) 2513 Mask <<= 2; 2514 } 2515 2516 return Mask; 2517} 2518 2519/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2520/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2521/// instructions. 2522unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2523 unsigned Mask = 0; 2524 // 8 nodes, but we only care about the first 4. 2525 for (int i = 3; i >= 0; --i) { 2526 unsigned Val = 0; 2527 SDOperand Arg = N->getOperand(i); 2528 if (Arg.getOpcode() != ISD::UNDEF) 2529 Val = cast<ConstantSDNode>(Arg)->getValue(); 2530 Mask |= Val; 2531 if (i != 0) 2532 Mask <<= 2; 2533 } 2534 2535 return Mask; 2536} 2537 2538/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2539/// specifies a 8 element shuffle that can be broken into a pair of 2540/// PSHUFHW and PSHUFLW. 2541static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2542 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2543 2544 if (N->getNumOperands() != 8) 2545 return false; 2546 2547 // Lower quadword shuffled. 2548 for (unsigned i = 0; i != 4; ++i) { 2549 SDOperand Arg = N->getOperand(i); 2550 if (Arg.getOpcode() == ISD::UNDEF) continue; 2551 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2552 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2553 if (Val >= 4) 2554 return false; 2555 } 2556 2557 // Upper quadword shuffled. 2558 for (unsigned i = 4; i != 8; ++i) { 2559 SDOperand Arg = N->getOperand(i); 2560 if (Arg.getOpcode() == ISD::UNDEF) continue; 2561 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2562 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2563 if (Val < 4 || Val > 7) 2564 return false; 2565 } 2566 2567 return true; 2568} 2569 2570/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2571/// values in ther permute mask. 2572static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1, 2573 SDOperand &V2, SDOperand &Mask, 2574 SelectionDAG &DAG) { 2575 MVT VT = Op.getValueType(); 2576 MVT MaskVT = Mask.getValueType(); 2577 MVT EltVT = MaskVT.getVectorElementType(); 2578 unsigned NumElems = Mask.getNumOperands(); 2579 SmallVector<SDOperand, 8> MaskVec; 2580 2581 for (unsigned i = 0; i != NumElems; ++i) { 2582 SDOperand Arg = Mask.getOperand(i); 2583 if (Arg.getOpcode() == ISD::UNDEF) { 2584 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2585 continue; 2586 } 2587 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2588 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2589 if (Val < NumElems) 2590 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2591 else 2592 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2593 } 2594 2595 std::swap(V1, V2); 2596 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2597 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2598} 2599 2600/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2601/// the two vector operands have swapped position. 2602static 2603SDOperand CommuteVectorShuffleMask(SDOperand Mask, SelectionDAG &DAG) { 2604 MVT MaskVT = Mask.getValueType(); 2605 MVT EltVT = MaskVT.getVectorElementType(); 2606 unsigned NumElems = Mask.getNumOperands(); 2607 SmallVector<SDOperand, 8> MaskVec; 2608 for (unsigned i = 0; i != NumElems; ++i) { 2609 SDOperand Arg = Mask.getOperand(i); 2610 if (Arg.getOpcode() == ISD::UNDEF) { 2611 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2612 continue; 2613 } 2614 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2615 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2616 if (Val < NumElems) 2617 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2618 else 2619 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2620 } 2621 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2622} 2623 2624 2625/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2626/// match movhlps. The lower half elements should come from upper half of 2627/// V1 (and in order), and the upper half elements should come from the upper 2628/// half of V2 (and in order). 2629static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2630 unsigned NumElems = Mask->getNumOperands(); 2631 if (NumElems != 4) 2632 return false; 2633 for (unsigned i = 0, e = 2; i != e; ++i) 2634 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2635 return false; 2636 for (unsigned i = 2; i != 4; ++i) 2637 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2638 return false; 2639 return true; 2640} 2641 2642/// isScalarLoadToVector - Returns true if the node is a scalar load that 2643/// is promoted to a vector. It also returns the LoadSDNode by reference if 2644/// required. 2645static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2646 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2647 N = N->getOperand(0).Val; 2648 if (ISD::isNON_EXTLoad(N)) { 2649 if (LD) 2650 *LD = cast<LoadSDNode>(N); 2651 return true; 2652 } 2653 } 2654 return false; 2655} 2656 2657/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2658/// match movlp{s|d}. The lower half elements should come from lower half of 2659/// V1 (and in order), and the upper half elements should come from the upper 2660/// half of V2 (and in order). And since V1 will become the source of the 2661/// MOVLP, it must be either a vector load or a scalar load to vector. 2662static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2663 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2664 return false; 2665 // Is V2 is a vector load, don't do this transformation. We will try to use 2666 // load folding shufps op. 2667 if (ISD::isNON_EXTLoad(V2)) 2668 return false; 2669 2670 unsigned NumElems = Mask->getNumOperands(); 2671 if (NumElems != 2 && NumElems != 4) 2672 return false; 2673 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2674 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2675 return false; 2676 for (unsigned i = NumElems/2; i != NumElems; ++i) 2677 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2678 return false; 2679 return true; 2680} 2681 2682/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2683/// all the same. 2684static bool isSplatVector(SDNode *N) { 2685 if (N->getOpcode() != ISD::BUILD_VECTOR) 2686 return false; 2687 2688 SDOperand SplatValue = N->getOperand(0); 2689 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2690 if (N->getOperand(i) != SplatValue) 2691 return false; 2692 return true; 2693} 2694 2695/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2696/// to an undef. 2697static bool isUndefShuffle(SDNode *N) { 2698 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2699 return false; 2700 2701 SDOperand V1 = N->getOperand(0); 2702 SDOperand V2 = N->getOperand(1); 2703 SDOperand Mask = N->getOperand(2); 2704 unsigned NumElems = Mask.getNumOperands(); 2705 for (unsigned i = 0; i != NumElems; ++i) { 2706 SDOperand Arg = Mask.getOperand(i); 2707 if (Arg.getOpcode() != ISD::UNDEF) { 2708 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2709 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2710 return false; 2711 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2712 return false; 2713 } 2714 } 2715 return true; 2716} 2717 2718/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2719/// constant +0.0. 2720static inline bool isZeroNode(SDOperand Elt) { 2721 return ((isa<ConstantSDNode>(Elt) && 2722 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2723 (isa<ConstantFPSDNode>(Elt) && 2724 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2725} 2726 2727/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2728/// to an zero vector. 2729static bool isZeroShuffle(SDNode *N) { 2730 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2731 return false; 2732 2733 SDOperand V1 = N->getOperand(0); 2734 SDOperand V2 = N->getOperand(1); 2735 SDOperand Mask = N->getOperand(2); 2736 unsigned NumElems = Mask.getNumOperands(); 2737 for (unsigned i = 0; i != NumElems; ++i) { 2738 SDOperand Arg = Mask.getOperand(i); 2739 if (Arg.getOpcode() == ISD::UNDEF) 2740 continue; 2741 2742 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2743 if (Idx < NumElems) { 2744 unsigned Opc = V1.Val->getOpcode(); 2745 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.Val)) 2746 continue; 2747 if (Opc != ISD::BUILD_VECTOR || 2748 !isZeroNode(V1.Val->getOperand(Idx))) 2749 return false; 2750 } else if (Idx >= NumElems) { 2751 unsigned Opc = V2.Val->getOpcode(); 2752 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.Val)) 2753 continue; 2754 if (Opc != ISD::BUILD_VECTOR || 2755 !isZeroNode(V2.Val->getOperand(Idx - NumElems))) 2756 return false; 2757 } 2758 } 2759 return true; 2760} 2761 2762/// getZeroVector - Returns a vector of specified type with all zero elements. 2763/// 2764static SDOperand getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2765 assert(VT.isVector() && "Expected a vector type"); 2766 2767 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2768 // type. This ensures they get CSE'd. 2769 SDOperand Vec; 2770 if (VT.getSizeInBits() == 64) { // MMX 2771 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2772 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2773 } else if (HasSSE2) { // SSE2 2774 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2775 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2776 } else { // SSE1 2777 SDOperand Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2778 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2779 } 2780 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2781} 2782 2783/// getOnesVector - Returns a vector of specified type with all bits set. 2784/// 2785static SDOperand getOnesVector(MVT VT, SelectionDAG &DAG) { 2786 assert(VT.isVector() && "Expected a vector type"); 2787 2788 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2789 // type. This ensures they get CSE'd. 2790 SDOperand Cst = DAG.getTargetConstant(~0U, MVT::i32); 2791 SDOperand Vec; 2792 if (VT.getSizeInBits() == 64) // MMX 2793 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2794 else // SSE 2795 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2796 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2797} 2798 2799 2800/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2801/// that point to V2 points to its first element. 2802static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) { 2803 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2804 2805 bool Changed = false; 2806 SmallVector<SDOperand, 8> MaskVec; 2807 unsigned NumElems = Mask.getNumOperands(); 2808 for (unsigned i = 0; i != NumElems; ++i) { 2809 SDOperand Arg = Mask.getOperand(i); 2810 if (Arg.getOpcode() != ISD::UNDEF) { 2811 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2812 if (Val > NumElems) { 2813 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2814 Changed = true; 2815 } 2816 } 2817 MaskVec.push_back(Arg); 2818 } 2819 2820 if (Changed) 2821 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2822 &MaskVec[0], MaskVec.size()); 2823 return Mask; 2824} 2825 2826/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2827/// operation of specified width. 2828static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2829 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2830 MVT BaseVT = MaskVT.getVectorElementType(); 2831 2832 SmallVector<SDOperand, 8> MaskVec; 2833 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2834 for (unsigned i = 1; i != NumElems; ++i) 2835 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2836 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2837} 2838 2839/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2840/// of specified width. 2841static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2842 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2843 MVT BaseVT = MaskVT.getVectorElementType(); 2844 SmallVector<SDOperand, 8> MaskVec; 2845 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2846 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2847 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2848 } 2849 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2850} 2851 2852/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2853/// of specified width. 2854static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2855 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2856 MVT BaseVT = MaskVT.getVectorElementType(); 2857 unsigned Half = NumElems/2; 2858 SmallVector<SDOperand, 8> MaskVec; 2859 for (unsigned i = 0; i != Half; ++i) { 2860 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2861 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2862 } 2863 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2864} 2865 2866/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2867/// element #0 of a vector with the specified index, leaving the rest of the 2868/// elements in place. 2869static SDOperand getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2870 SelectionDAG &DAG) { 2871 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2872 MVT BaseVT = MaskVT.getVectorElementType(); 2873 SmallVector<SDOperand, 8> MaskVec; 2874 // Element #0 of the result gets the elt we are replacing. 2875 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2876 for (unsigned i = 1; i != NumElems; ++i) 2877 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2878 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2879} 2880 2881/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2882static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) { 2883 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2884 MVT VT = Op.getValueType(); 2885 if (PVT == VT) 2886 return Op; 2887 SDOperand V1 = Op.getOperand(0); 2888 SDOperand Mask = Op.getOperand(2); 2889 unsigned NumElems = Mask.getNumOperands(); 2890 // Special handling of v4f32 -> v4i32. 2891 if (VT != MVT::v4f32) { 2892 Mask = getUnpacklMask(NumElems, DAG); 2893 while (NumElems > 4) { 2894 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2895 NumElems >>= 1; 2896 } 2897 Mask = getZeroVector(MVT::v4i32, true, DAG); 2898 } 2899 2900 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2901 SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2902 DAG.getNode(ISD::UNDEF, PVT), Mask); 2903 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2904} 2905 2906/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2907/// vector of zero or undef vector. This produces a shuffle where the low 2908/// element of V2 is swizzled into the zero/undef vector, landing at element 2909/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2910static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx, 2911 bool isZero, bool HasSSE2, 2912 SelectionDAG &DAG) { 2913 MVT VT = V2.getValueType(); 2914 SDOperand V1 = isZero 2915 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2916 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2917 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2918 MVT EVT = MaskVT.getVectorElementType(); 2919 SmallVector<SDOperand, 16> MaskVec; 2920 for (unsigned i = 0; i != NumElems; ++i) 2921 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2922 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2923 else 2924 MaskVec.push_back(DAG.getConstant(i, EVT)); 2925 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2926 &MaskVec[0], MaskVec.size()); 2927 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2928} 2929 2930/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2931/// a shuffle that is zero. 2932static 2933unsigned getNumOfConsecutiveZeros(SDOperand Op, SDOperand Mask, 2934 unsigned NumElems, bool Low, 2935 SelectionDAG &DAG) { 2936 unsigned NumZeros = 0; 2937 for (unsigned i = 0; i < NumElems; ++i) { 2938 unsigned Index = Low ? i : NumElems-i-1; 2939 SDOperand Idx = Mask.getOperand(Index); 2940 if (Idx.getOpcode() == ISD::UNDEF) { 2941 ++NumZeros; 2942 continue; 2943 } 2944 SDOperand Elt = DAG.getShuffleScalarElt(Op.Val, Index); 2945 if (Elt.Val && isZeroNode(Elt)) 2946 ++NumZeros; 2947 else 2948 break; 2949 } 2950 return NumZeros; 2951} 2952 2953/// isVectorShift - Returns true if the shuffle can be implemented as a 2954/// logical left or right shift of a vector. 2955static bool isVectorShift(SDOperand Op, SDOperand Mask, SelectionDAG &DAG, 2956 bool &isLeft, SDOperand &ShVal, unsigned &ShAmt) { 2957 unsigned NumElems = Mask.getNumOperands(); 2958 2959 isLeft = true; 2960 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 2961 if (!NumZeros) { 2962 isLeft = false; 2963 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 2964 if (!NumZeros) 2965 return false; 2966 } 2967 2968 bool SeenV1 = false; 2969 bool SeenV2 = false; 2970 for (unsigned i = NumZeros; i < NumElems; ++i) { 2971 unsigned Val = isLeft ? (i - NumZeros) : i; 2972 SDOperand Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 2973 if (Idx.getOpcode() == ISD::UNDEF) 2974 continue; 2975 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 2976 if (Index < NumElems) 2977 SeenV1 = true; 2978 else { 2979 Index -= NumElems; 2980 SeenV2 = true; 2981 } 2982 if (Index != Val) 2983 return false; 2984 } 2985 if (SeenV1 && SeenV2) 2986 return false; 2987 2988 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 2989 ShAmt = NumZeros; 2990 return true; 2991} 2992 2993 2994/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 2995/// 2996static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros, 2997 unsigned NumNonZero, unsigned NumZero, 2998 SelectionDAG &DAG, TargetLowering &TLI) { 2999 if (NumNonZero > 8) 3000 return SDOperand(); 3001 3002 SDOperand V(0, 0); 3003 bool First = true; 3004 for (unsigned i = 0; i < 16; ++i) { 3005 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3006 if (ThisIsNonZero && First) { 3007 if (NumZero) 3008 V = getZeroVector(MVT::v8i16, true, DAG); 3009 else 3010 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3011 First = false; 3012 } 3013 3014 if ((i & 1) != 0) { 3015 SDOperand ThisElt(0, 0), LastElt(0, 0); 3016 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3017 if (LastIsNonZero) { 3018 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3019 } 3020 if (ThisIsNonZero) { 3021 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3022 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3023 ThisElt, DAG.getConstant(8, MVT::i8)); 3024 if (LastIsNonZero) 3025 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3026 } else 3027 ThisElt = LastElt; 3028 3029 if (ThisElt.Val) 3030 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3031 DAG.getIntPtrConstant(i/2)); 3032 } 3033 } 3034 3035 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3036} 3037 3038/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3039/// 3040static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros, 3041 unsigned NumNonZero, unsigned NumZero, 3042 SelectionDAG &DAG, TargetLowering &TLI) { 3043 if (NumNonZero > 4) 3044 return SDOperand(); 3045 3046 SDOperand V(0, 0); 3047 bool First = true; 3048 for (unsigned i = 0; i < 8; ++i) { 3049 bool isNonZero = (NonZeros & (1 << i)) != 0; 3050 if (isNonZero) { 3051 if (First) { 3052 if (NumZero) 3053 V = getZeroVector(MVT::v8i16, true, DAG); 3054 else 3055 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3056 First = false; 3057 } 3058 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3059 DAG.getIntPtrConstant(i)); 3060 } 3061 } 3062 3063 return V; 3064} 3065 3066/// getVShift - Return a vector logical shift node. 3067/// 3068static SDOperand getVShift(bool isLeft, MVT VT, SDOperand SrcOp, 3069 unsigned NumBits, SelectionDAG &DAG, 3070 const TargetLowering &TLI) { 3071 bool isMMX = VT.getSizeInBits() == 64; 3072 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3073 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3074 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3075 return DAG.getNode(ISD::BIT_CONVERT, VT, 3076 DAG.getNode(Opc, ShVT, SrcOp, 3077 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3078} 3079 3080SDOperand 3081X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) { 3082 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3083 if (ISD::isBuildVectorAllZeros(Op.Val) || ISD::isBuildVectorAllOnes(Op.Val)) { 3084 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3085 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3086 // eliminated on x86-32 hosts. 3087 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3088 return Op; 3089 3090 if (ISD::isBuildVectorAllOnes(Op.Val)) 3091 return getOnesVector(Op.getValueType(), DAG); 3092 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3093 } 3094 3095 MVT VT = Op.getValueType(); 3096 MVT EVT = VT.getVectorElementType(); 3097 unsigned EVTBits = EVT.getSizeInBits(); 3098 3099 unsigned NumElems = Op.getNumOperands(); 3100 unsigned NumZero = 0; 3101 unsigned NumNonZero = 0; 3102 unsigned NonZeros = 0; 3103 bool IsAllConstants = true; 3104 SmallSet<SDOperand, 8> Values; 3105 for (unsigned i = 0; i < NumElems; ++i) { 3106 SDOperand Elt = Op.getOperand(i); 3107 if (Elt.getOpcode() == ISD::UNDEF) 3108 continue; 3109 Values.insert(Elt); 3110 if (Elt.getOpcode() != ISD::Constant && 3111 Elt.getOpcode() != ISD::ConstantFP) 3112 IsAllConstants = false; 3113 if (isZeroNode(Elt)) 3114 NumZero++; 3115 else { 3116 NonZeros |= (1 << i); 3117 NumNonZero++; 3118 } 3119 } 3120 3121 if (NumNonZero == 0) { 3122 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3123 return DAG.getNode(ISD::UNDEF, VT); 3124 } 3125 3126 // Special case for single non-zero, non-undef, element. 3127 if (NumNonZero == 1 && NumElems <= 4) { 3128 unsigned Idx = CountTrailingZeros_32(NonZeros); 3129 SDOperand Item = Op.getOperand(Idx); 3130 3131 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3132 // the value are obviously zero, truncate the value to i32 and do the 3133 // insertion that way. Only do this if the value is non-constant or if the 3134 // value is a constant being inserted into element 0. It is cheaper to do 3135 // a constant pool load than it is to do a movd + shuffle. 3136 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3137 (!IsAllConstants || Idx == 0)) { 3138 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3139 // Handle MMX and SSE both. 3140 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3141 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3142 3143 // Truncate the value (which may itself be a constant) to i32, and 3144 // convert it to a vector with movd (S2V+shuffle to zero extend). 3145 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3146 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3147 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3148 Subtarget->hasSSE2(), DAG); 3149 3150 // Now we have our 32-bit value zero extended in the low element of 3151 // a vector. If Idx != 0, swizzle it into place. 3152 if (Idx != 0) { 3153 SDOperand Ops[] = { 3154 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3155 getSwapEltZeroMask(VecElts, Idx, DAG) 3156 }; 3157 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3158 } 3159 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3160 } 3161 } 3162 3163 // If we have a constant or non-constant insertion into the low element of 3164 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3165 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3166 // depending on what the source datatype is. Because we can only get here 3167 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3168 if (Idx == 0 && 3169 // Don't do this for i64 values on x86-32. 3170 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3171 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3172 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3173 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3174 Subtarget->hasSSE2(), DAG); 3175 } 3176 3177 // Is it a vector logical left shift? 3178 if (NumElems == 2 && Idx == 1 && 3179 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3180 unsigned NumBits = VT.getSizeInBits(); 3181 return getVShift(true, VT, 3182 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3183 NumBits/2, DAG, *this); 3184 } 3185 3186 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3187 return SDOperand(); 3188 3189 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3190 // is a non-constant being inserted into an element other than the low one, 3191 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3192 // movd/movss) to move this into the low element, then shuffle it into 3193 // place. 3194 if (EVTBits == 32) { 3195 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3196 3197 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3198 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3199 Subtarget->hasSSE2(), DAG); 3200 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3201 MVT MaskEVT = MaskVT.getVectorElementType(); 3202 SmallVector<SDOperand, 8> MaskVec; 3203 for (unsigned i = 0; i < NumElems; i++) 3204 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3205 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3206 &MaskVec[0], MaskVec.size()); 3207 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3208 DAG.getNode(ISD::UNDEF, VT), Mask); 3209 } 3210 } 3211 3212 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3213 if (Values.size() == 1) 3214 return SDOperand(); 3215 3216 // A vector full of immediates; various special cases are already 3217 // handled, so this is best done with a single constant-pool load. 3218 if (IsAllConstants) 3219 return SDOperand(); 3220 3221 // Let legalizer expand 2-wide build_vectors. 3222 if (EVTBits == 64) { 3223 if (NumNonZero == 1) { 3224 // One half is zero or undef. 3225 unsigned Idx = CountTrailingZeros_32(NonZeros); 3226 SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3227 Op.getOperand(Idx)); 3228 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3229 Subtarget->hasSSE2(), DAG); 3230 } 3231 return SDOperand(); 3232 } 3233 3234 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3235 if (EVTBits == 8 && NumElems == 16) { 3236 SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3237 *this); 3238 if (V.Val) return V; 3239 } 3240 3241 if (EVTBits == 16 && NumElems == 8) { 3242 SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3243 *this); 3244 if (V.Val) return V; 3245 } 3246 3247 // If element VT is == 32 bits, turn it into a number of shuffles. 3248 SmallVector<SDOperand, 8> V; 3249 V.resize(NumElems); 3250 if (NumElems == 4 && NumZero > 0) { 3251 for (unsigned i = 0; i < 4; ++i) { 3252 bool isZero = !(NonZeros & (1 << i)); 3253 if (isZero) 3254 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3255 else 3256 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3257 } 3258 3259 for (unsigned i = 0; i < 2; ++i) { 3260 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3261 default: break; 3262 case 0: 3263 V[i] = V[i*2]; // Must be a zero vector. 3264 break; 3265 case 1: 3266 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3267 getMOVLMask(NumElems, DAG)); 3268 break; 3269 case 2: 3270 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3271 getMOVLMask(NumElems, DAG)); 3272 break; 3273 case 3: 3274 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3275 getUnpacklMask(NumElems, DAG)); 3276 break; 3277 } 3278 } 3279 3280 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3281 MVT EVT = MaskVT.getVectorElementType(); 3282 SmallVector<SDOperand, 8> MaskVec; 3283 bool Reverse = (NonZeros & 0x3) == 2; 3284 for (unsigned i = 0; i < 2; ++i) 3285 if (Reverse) 3286 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3287 else 3288 MaskVec.push_back(DAG.getConstant(i, EVT)); 3289 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3290 for (unsigned i = 0; i < 2; ++i) 3291 if (Reverse) 3292 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3293 else 3294 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3295 SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3296 &MaskVec[0], MaskVec.size()); 3297 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3298 } 3299 3300 if (Values.size() > 2) { 3301 // Expand into a number of unpckl*. 3302 // e.g. for v4f32 3303 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3304 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3305 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3306 SDOperand UnpckMask = getUnpacklMask(NumElems, DAG); 3307 for (unsigned i = 0; i < NumElems; ++i) 3308 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3309 NumElems >>= 1; 3310 while (NumElems != 0) { 3311 for (unsigned i = 0; i < NumElems; ++i) 3312 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3313 UnpckMask); 3314 NumElems >>= 1; 3315 } 3316 return V[0]; 3317 } 3318 3319 return SDOperand(); 3320} 3321 3322static 3323SDOperand LowerVECTOR_SHUFFLEv8i16(SDOperand V1, SDOperand V2, 3324 SDOperand PermMask, SelectionDAG &DAG, 3325 TargetLowering &TLI) { 3326 SDOperand NewV; 3327 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3328 MVT MaskEVT = MaskVT.getVectorElementType(); 3329 MVT PtrVT = TLI.getPointerTy(); 3330 SmallVector<SDOperand, 8> MaskElts(PermMask.Val->op_begin(), 3331 PermMask.Val->op_end()); 3332 3333 // First record which half of which vector the low elements come from. 3334 SmallVector<unsigned, 4> LowQuad(4); 3335 for (unsigned i = 0; i < 4; ++i) { 3336 SDOperand Elt = MaskElts[i]; 3337 if (Elt.getOpcode() == ISD::UNDEF) 3338 continue; 3339 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3340 int QuadIdx = EltIdx / 4; 3341 ++LowQuad[QuadIdx]; 3342 } 3343 int BestLowQuad = -1; 3344 unsigned MaxQuad = 1; 3345 for (unsigned i = 0; i < 4; ++i) { 3346 if (LowQuad[i] > MaxQuad) { 3347 BestLowQuad = i; 3348 MaxQuad = LowQuad[i]; 3349 } 3350 } 3351 3352 // Record which half of which vector the high elements come from. 3353 SmallVector<unsigned, 4> HighQuad(4); 3354 for (unsigned i = 4; i < 8; ++i) { 3355 SDOperand Elt = MaskElts[i]; 3356 if (Elt.getOpcode() == ISD::UNDEF) 3357 continue; 3358 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3359 int QuadIdx = EltIdx / 4; 3360 ++HighQuad[QuadIdx]; 3361 } 3362 int BestHighQuad = -1; 3363 MaxQuad = 1; 3364 for (unsigned i = 0; i < 4; ++i) { 3365 if (HighQuad[i] > MaxQuad) { 3366 BestHighQuad = i; 3367 MaxQuad = HighQuad[i]; 3368 } 3369 } 3370 3371 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3372 if (BestLowQuad != -1 || BestHighQuad != -1) { 3373 // First sort the 4 chunks in order using shufpd. 3374 SmallVector<SDOperand, 8> MaskVec; 3375 if (BestLowQuad != -1) 3376 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3377 else 3378 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3379 if (BestHighQuad != -1) 3380 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3381 else 3382 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3383 SDOperand Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3384 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3385 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3386 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3387 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3388 3389 // Now sort high and low parts separately. 3390 BitVector InOrder(8); 3391 if (BestLowQuad != -1) { 3392 // Sort lower half in order using PSHUFLW. 3393 MaskVec.clear(); 3394 bool AnyOutOrder = false; 3395 for (unsigned i = 0; i != 4; ++i) { 3396 SDOperand Elt = MaskElts[i]; 3397 if (Elt.getOpcode() == ISD::UNDEF) { 3398 MaskVec.push_back(Elt); 3399 InOrder.set(i); 3400 } else { 3401 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3402 if (EltIdx != i) 3403 AnyOutOrder = true; 3404 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3405 // If this element is in the right place after this shuffle, then 3406 // remember it. 3407 if ((int)(EltIdx / 4) == BestLowQuad) 3408 InOrder.set(i); 3409 } 3410 } 3411 if (AnyOutOrder) { 3412 for (unsigned i = 4; i != 8; ++i) 3413 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3414 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3415 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3416 } 3417 } 3418 3419 if (BestHighQuad != -1) { 3420 // Sort high half in order using PSHUFHW if possible. 3421 MaskVec.clear(); 3422 for (unsigned i = 0; i != 4; ++i) 3423 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3424 bool AnyOutOrder = false; 3425 for (unsigned i = 4; i != 8; ++i) { 3426 SDOperand Elt = MaskElts[i]; 3427 if (Elt.getOpcode() == ISD::UNDEF) { 3428 MaskVec.push_back(Elt); 3429 InOrder.set(i); 3430 } else { 3431 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3432 if (EltIdx != i) 3433 AnyOutOrder = true; 3434 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3435 // If this element is in the right place after this shuffle, then 3436 // remember it. 3437 if ((int)(EltIdx / 4) == BestHighQuad) 3438 InOrder.set(i); 3439 } 3440 } 3441 if (AnyOutOrder) { 3442 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3443 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3444 } 3445 } 3446 3447 // The other elements are put in the right place using pextrw and pinsrw. 3448 for (unsigned i = 0; i != 8; ++i) { 3449 if (InOrder[i]) 3450 continue; 3451 SDOperand Elt = MaskElts[i]; 3452 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3453 SDOperand ExtOp = (EltIdx < 8) 3454 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3455 DAG.getConstant(EltIdx, PtrVT)) 3456 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3457 DAG.getConstant(EltIdx - 8, PtrVT)); 3458 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3459 DAG.getConstant(i, PtrVT)); 3460 } 3461 return NewV; 3462 } 3463 3464 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use 3465 ///as few as possible. 3466 // First, let's find out how many elements are already in the right order. 3467 unsigned V1InOrder = 0; 3468 unsigned V1FromV1 = 0; 3469 unsigned V2InOrder = 0; 3470 unsigned V2FromV2 = 0; 3471 SmallVector<SDOperand, 8> V1Elts; 3472 SmallVector<SDOperand, 8> V2Elts; 3473 for (unsigned i = 0; i < 8; ++i) { 3474 SDOperand Elt = MaskElts[i]; 3475 if (Elt.getOpcode() == ISD::UNDEF) { 3476 V1Elts.push_back(Elt); 3477 V2Elts.push_back(Elt); 3478 ++V1InOrder; 3479 ++V2InOrder; 3480 continue; 3481 } 3482 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3483 if (EltIdx == i) { 3484 V1Elts.push_back(Elt); 3485 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3486 ++V1InOrder; 3487 } else if (EltIdx == i+8) { 3488 V1Elts.push_back(Elt); 3489 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3490 ++V2InOrder; 3491 } else if (EltIdx < 8) { 3492 V1Elts.push_back(Elt); 3493 ++V1FromV1; 3494 } else { 3495 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3496 ++V2FromV2; 3497 } 3498 } 3499 3500 if (V2InOrder > V1InOrder) { 3501 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3502 std::swap(V1, V2); 3503 std::swap(V1Elts, V2Elts); 3504 std::swap(V1FromV1, V2FromV2); 3505 } 3506 3507 if ((V1FromV1 + V1InOrder) != 8) { 3508 // Some elements are from V2. 3509 if (V1FromV1) { 3510 // If there are elements that are from V1 but out of place, 3511 // then first sort them in place 3512 SmallVector<SDOperand, 8> MaskVec; 3513 for (unsigned i = 0; i < 8; ++i) { 3514 SDOperand Elt = V1Elts[i]; 3515 if (Elt.getOpcode() == ISD::UNDEF) { 3516 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3517 continue; 3518 } 3519 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3520 if (EltIdx >= 8) 3521 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3522 else 3523 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3524 } 3525 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3526 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3527 } 3528 3529 NewV = V1; 3530 for (unsigned i = 0; i < 8; ++i) { 3531 SDOperand Elt = V1Elts[i]; 3532 if (Elt.getOpcode() == ISD::UNDEF) 3533 continue; 3534 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3535 if (EltIdx < 8) 3536 continue; 3537 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3538 DAG.getConstant(EltIdx - 8, PtrVT)); 3539 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3540 DAG.getConstant(i, PtrVT)); 3541 } 3542 return NewV; 3543 } else { 3544 // All elements are from V1. 3545 NewV = V1; 3546 for (unsigned i = 0; i < 8; ++i) { 3547 SDOperand Elt = V1Elts[i]; 3548 if (Elt.getOpcode() == ISD::UNDEF) 3549 continue; 3550 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3551 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3552 DAG.getConstant(EltIdx, PtrVT)); 3553 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3554 DAG.getConstant(i, PtrVT)); 3555 } 3556 return NewV; 3557 } 3558} 3559 3560/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3561/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3562/// done when every pair / quad of shuffle mask elements point to elements in 3563/// the right sequence. e.g. 3564/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3565static 3566SDOperand RewriteAsNarrowerShuffle(SDOperand V1, SDOperand V2, 3567 MVT VT, 3568 SDOperand PermMask, SelectionDAG &DAG, 3569 TargetLowering &TLI) { 3570 unsigned NumElems = PermMask.getNumOperands(); 3571 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3572 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3573 MVT MaskEltVT = MaskVT.getVectorElementType(); 3574 MVT NewVT = MaskVT; 3575 switch (VT.getSimpleVT()) { 3576 default: assert(false && "Unexpected!"); 3577 case MVT::v4f32: NewVT = MVT::v2f64; break; 3578 case MVT::v4i32: NewVT = MVT::v2i64; break; 3579 case MVT::v8i16: NewVT = MVT::v4i32; break; 3580 case MVT::v16i8: NewVT = MVT::v4i32; break; 3581 } 3582 3583 if (NewWidth == 2) { 3584 if (VT.isInteger()) 3585 NewVT = MVT::v2i64; 3586 else 3587 NewVT = MVT::v2f64; 3588 } 3589 unsigned Scale = NumElems / NewWidth; 3590 SmallVector<SDOperand, 8> MaskVec; 3591 for (unsigned i = 0; i < NumElems; i += Scale) { 3592 unsigned StartIdx = ~0U; 3593 for (unsigned j = 0; j < Scale; ++j) { 3594 SDOperand Elt = PermMask.getOperand(i+j); 3595 if (Elt.getOpcode() == ISD::UNDEF) 3596 continue; 3597 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3598 if (StartIdx == ~0U) 3599 StartIdx = EltIdx - (EltIdx % Scale); 3600 if (EltIdx != StartIdx + j) 3601 return SDOperand(); 3602 } 3603 if (StartIdx == ~0U) 3604 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEltVT)); 3605 else 3606 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MaskEltVT)); 3607 } 3608 3609 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3610 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3611 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3612 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3613 &MaskVec[0], MaskVec.size())); 3614} 3615 3616/// getVZextMovL - Return a zero-extending vector move low node. 3617/// 3618static SDOperand getVZextMovL(MVT VT, MVT OpVT, 3619 SDOperand SrcOp, SelectionDAG &DAG, 3620 const X86Subtarget *Subtarget) { 3621 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3622 LoadSDNode *LD = NULL; 3623 if (!isScalarLoadToVector(SrcOp.Val, &LD)) 3624 LD = dyn_cast<LoadSDNode>(SrcOp); 3625 if (!LD) { 3626 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3627 // instead. 3628 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3629 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3630 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3631 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3632 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3633 // PR2108 3634 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3635 return DAG.getNode(ISD::BIT_CONVERT, VT, 3636 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3637 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3638 SrcOp.getOperand(0).getOperand(0)))); 3639 } 3640 } 3641 } 3642 3643 return DAG.getNode(ISD::BIT_CONVERT, VT, 3644 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3645 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3646} 3647 3648/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of 3649/// shuffles. 3650static SDOperand 3651LowerVECTOR_SHUFFLE_4wide(SDOperand V1, SDOperand V2, 3652 SDOperand PermMask, MVT VT, SelectionDAG &DAG) { 3653 MVT MaskVT = PermMask.getValueType(); 3654 MVT MaskEVT = MaskVT.getVectorElementType(); 3655 SmallVector<std::pair<int, int>, 8> Locs; 3656 Locs.reserve(4); 3657 SmallVector<SDOperand, 8> Mask1(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3658 unsigned NumHi = 0; 3659 unsigned NumLo = 0; 3660 for (unsigned i = 0; i != 4; ++i) { 3661 SDOperand Elt = PermMask.getOperand(i); 3662 if (Elt.getOpcode() == ISD::UNDEF) { 3663 Locs[i] = std::make_pair(-1, -1); 3664 } else { 3665 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3666 if (Val < 4) { 3667 Locs[i] = std::make_pair(0, NumLo); 3668 Mask1[NumLo] = Elt; 3669 NumLo++; 3670 } else { 3671 Locs[i] = std::make_pair(1, NumHi); 3672 if (2+NumHi < 4) 3673 Mask1[2+NumHi] = Elt; 3674 NumHi++; 3675 } 3676 } 3677 } 3678 3679 if (NumLo <= 2 && NumHi <= 2) { 3680 // If no more than two elements come from either vector. This can be 3681 // implemented with two shuffles. First shuffle gather the elements. 3682 // The second shuffle, which takes the first shuffle as both of its 3683 // vector operands, put the elements into the right order. 3684 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3685 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3686 &Mask1[0], Mask1.size())); 3687 3688 SmallVector<SDOperand, 8> Mask2(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3689 for (unsigned i = 0; i != 4; ++i) { 3690 if (Locs[i].first == -1) 3691 continue; 3692 else { 3693 unsigned Idx = (i < 2) ? 0 : 4; 3694 Idx += Locs[i].first * 2 + Locs[i].second; 3695 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3696 } 3697 } 3698 3699 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3700 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3701 &Mask2[0], Mask2.size())); 3702 } else if (NumLo == 3 || NumHi == 3) { 3703 // Otherwise, we must have three elements from one vector, call it X, and 3704 // one element from the other, call it Y. First, use a shufps to build an 3705 // intermediate vector with the one element from Y and the element from X 3706 // that will be in the same half in the final destination (the indexes don't 3707 // matter). Then, use a shufps to build the final vector, taking the half 3708 // containing the element from Y from the intermediate, and the other half 3709 // from X. 3710 if (NumHi == 3) { 3711 // Normalize it so the 3 elements come from V1. 3712 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3713 std::swap(V1, V2); 3714 } 3715 3716 // Find the element from V2. 3717 unsigned HiIndex; 3718 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 3719 SDOperand Elt = PermMask.getOperand(HiIndex); 3720 if (Elt.getOpcode() == ISD::UNDEF) 3721 continue; 3722 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3723 if (Val >= 4) 3724 break; 3725 } 3726 3727 Mask1[0] = PermMask.getOperand(HiIndex); 3728 Mask1[1] = DAG.getNode(ISD::UNDEF, MaskEVT); 3729 Mask1[2] = PermMask.getOperand(HiIndex^1); 3730 Mask1[3] = DAG.getNode(ISD::UNDEF, MaskEVT); 3731 V2 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3732 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3733 3734 if (HiIndex >= 2) { 3735 Mask1[0] = PermMask.getOperand(0); 3736 Mask1[1] = PermMask.getOperand(1); 3737 Mask1[2] = DAG.getConstant(HiIndex & 1 ? 6 : 4, MaskEVT); 3738 Mask1[3] = DAG.getConstant(HiIndex & 1 ? 4 : 6, MaskEVT); 3739 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3740 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3741 } else { 3742 Mask1[0] = DAG.getConstant(HiIndex & 1 ? 2 : 0, MaskEVT); 3743 Mask1[1] = DAG.getConstant(HiIndex & 1 ? 0 : 2, MaskEVT); 3744 Mask1[2] = PermMask.getOperand(2); 3745 Mask1[3] = PermMask.getOperand(3); 3746 if (Mask1[2].getOpcode() != ISD::UNDEF) 3747 Mask1[2] = DAG.getConstant(cast<ConstantSDNode>(Mask1[2])->getValue()+4, 3748 MaskEVT); 3749 if (Mask1[3].getOpcode() != ISD::UNDEF) 3750 Mask1[3] = DAG.getConstant(cast<ConstantSDNode>(Mask1[3])->getValue()+4, 3751 MaskEVT); 3752 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, 3753 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3754 } 3755 } 3756 3757 // Break it into (shuffle shuffle_hi, shuffle_lo). 3758 Locs.clear(); 3759 SmallVector<SDOperand,8> LoMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3760 SmallVector<SDOperand,8> HiMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3761 SmallVector<SDOperand,8> *MaskPtr = &LoMask; 3762 unsigned MaskIdx = 0; 3763 unsigned LoIdx = 0; 3764 unsigned HiIdx = 2; 3765 for (unsigned i = 0; i != 4; ++i) { 3766 if (i == 2) { 3767 MaskPtr = &HiMask; 3768 MaskIdx = 1; 3769 LoIdx = 0; 3770 HiIdx = 2; 3771 } 3772 SDOperand Elt = PermMask.getOperand(i); 3773 if (Elt.getOpcode() == ISD::UNDEF) { 3774 Locs[i] = std::make_pair(-1, -1); 3775 } else if (cast<ConstantSDNode>(Elt)->getValue() < 4) { 3776 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3777 (*MaskPtr)[LoIdx] = Elt; 3778 LoIdx++; 3779 } else { 3780 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3781 (*MaskPtr)[HiIdx] = Elt; 3782 HiIdx++; 3783 } 3784 } 3785 3786 SDOperand LoShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3787 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3788 &LoMask[0], LoMask.size())); 3789 SDOperand HiShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3790 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3791 &HiMask[0], HiMask.size())); 3792 SmallVector<SDOperand, 8> MaskOps; 3793 for (unsigned i = 0; i != 4; ++i) { 3794 if (Locs[i].first == -1) { 3795 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3796 } else { 3797 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 3798 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3799 } 3800 } 3801 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3802 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3803 &MaskOps[0], MaskOps.size())); 3804} 3805 3806SDOperand 3807X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) { 3808 SDOperand V1 = Op.getOperand(0); 3809 SDOperand V2 = Op.getOperand(1); 3810 SDOperand PermMask = Op.getOperand(2); 3811 MVT VT = Op.getValueType(); 3812 unsigned NumElems = PermMask.getNumOperands(); 3813 bool isMMX = VT.getSizeInBits() == 64; 3814 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3815 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3816 bool V1IsSplat = false; 3817 bool V2IsSplat = false; 3818 3819 if (isUndefShuffle(Op.Val)) 3820 return DAG.getNode(ISD::UNDEF, VT); 3821 3822 if (isZeroShuffle(Op.Val)) 3823 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3824 3825 if (isIdentityMask(PermMask.Val)) 3826 return V1; 3827 else if (isIdentityMask(PermMask.Val, true)) 3828 return V2; 3829 3830 if (isSplatMask(PermMask.Val)) { 3831 if (isMMX || NumElems < 4) return Op; 3832 // Promote it to a v4{if}32 splat. 3833 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3834 } 3835 3836 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3837 // do it! 3838 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3839 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3840 if (NewOp.Val) 3841 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3842 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3843 // FIXME: Figure out a cleaner way to do this. 3844 // Try to make use of movq to zero out the top part. 3845 if (ISD::isBuildVectorAllZeros(V2.Val)) { 3846 SDOperand NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3847 DAG, *this); 3848 if (NewOp.Val) { 3849 SDOperand NewV1 = NewOp.getOperand(0); 3850 SDOperand NewV2 = NewOp.getOperand(1); 3851 SDOperand NewMask = NewOp.getOperand(2); 3852 if (isCommutedMOVL(NewMask.Val, true, false)) { 3853 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3854 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3855 } 3856 } 3857 } else if (ISD::isBuildVectorAllZeros(V1.Val)) { 3858 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3859 DAG, *this); 3860 if (NewOp.Val && X86::isMOVLMask(NewOp.getOperand(2).Val)) 3861 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3862 DAG, Subtarget); 3863 } 3864 } 3865 3866 // Check if this can be converted into a logical shift. 3867 bool isLeft = false; 3868 unsigned ShAmt = 0; 3869 SDOperand ShVal; 3870 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3871 if (isShift && ShVal.hasOneUse()) { 3872 // If the shifted value has multiple uses, it may be cheaper to use 3873 // v_set0 + movlhps or movhlps, etc. 3874 MVT EVT = VT.getVectorElementType(); 3875 ShAmt *= EVT.getSizeInBits(); 3876 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3877 } 3878 3879 if (X86::isMOVLMask(PermMask.Val)) { 3880 if (V1IsUndef) 3881 return V2; 3882 if (ISD::isBuildVectorAllZeros(V1.Val)) 3883 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3884 if (!isMMX) 3885 return Op; 3886 } 3887 3888 if (!isMMX && (X86::isMOVSHDUPMask(PermMask.Val) || 3889 X86::isMOVSLDUPMask(PermMask.Val) || 3890 X86::isMOVHLPSMask(PermMask.Val) || 3891 X86::isMOVHPMask(PermMask.Val) || 3892 X86::isMOVLPMask(PermMask.Val))) 3893 return Op; 3894 3895 if (ShouldXformToMOVHLPS(PermMask.Val) || 3896 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val)) 3897 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3898 3899 if (isShift) { 3900 // No better options. Use a vshl / vsrl. 3901 MVT EVT = VT.getVectorElementType(); 3902 ShAmt *= EVT.getSizeInBits(); 3903 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3904 } 3905 3906 bool Commuted = false; 3907 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3908 // 1,1,1,1 -> v8i16 though. 3909 V1IsSplat = isSplatVector(V1.Val); 3910 V2IsSplat = isSplatVector(V2.Val); 3911 3912 // Canonicalize the splat or undef, if present, to be on the RHS. 3913 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3914 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3915 std::swap(V1IsSplat, V2IsSplat); 3916 std::swap(V1IsUndef, V2IsUndef); 3917 Commuted = true; 3918 } 3919 3920 // FIXME: Figure out a cleaner way to do this. 3921 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) { 3922 if (V2IsUndef) return V1; 3923 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3924 if (V2IsSplat) { 3925 // V2 is a splat, so the mask may be malformed. That is, it may point 3926 // to any V2 element. The instruction selectior won't like this. Get 3927 // a corrected mask and commute to form a proper MOVS{S|D}. 3928 SDOperand NewMask = getMOVLMask(NumElems, DAG); 3929 if (NewMask.Val != PermMask.Val) 3930 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3931 } 3932 return Op; 3933 } 3934 3935 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3936 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3937 X86::isUNPCKLMask(PermMask.Val) || 3938 X86::isUNPCKHMask(PermMask.Val)) 3939 return Op; 3940 3941 if (V2IsSplat) { 3942 // Normalize mask so all entries that point to V2 points to its first 3943 // element then try to match unpck{h|l} again. If match, return a 3944 // new vector_shuffle with the corrected mask. 3945 SDOperand NewMask = NormalizeMask(PermMask, DAG); 3946 if (NewMask.Val != PermMask.Val) { 3947 if (X86::isUNPCKLMask(PermMask.Val, true)) { 3948 SDOperand NewMask = getUnpacklMask(NumElems, DAG); 3949 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3950 } else if (X86::isUNPCKHMask(PermMask.Val, true)) { 3951 SDOperand NewMask = getUnpackhMask(NumElems, DAG); 3952 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3953 } 3954 } 3955 } 3956 3957 // Normalize the node to match x86 shuffle ops if needed 3958 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val)) 3959 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3960 3961 if (Commuted) { 3962 // Commute is back and try unpck* again. 3963 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3964 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3965 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3966 X86::isUNPCKLMask(PermMask.Val) || 3967 X86::isUNPCKHMask(PermMask.Val)) 3968 return Op; 3969 } 3970 3971 // Try PSHUF* first, then SHUFP*. 3972 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 3973 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 3974 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.Val)) { 3975 if (V2.getOpcode() != ISD::UNDEF) 3976 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 3977 DAG.getNode(ISD::UNDEF, VT), PermMask); 3978 return Op; 3979 } 3980 3981 if (!isMMX) { 3982 if (Subtarget->hasSSE2() && 3983 (X86::isPSHUFDMask(PermMask.Val) || 3984 X86::isPSHUFHWMask(PermMask.Val) || 3985 X86::isPSHUFLWMask(PermMask.Val))) { 3986 MVT RVT = VT; 3987 if (VT == MVT::v4f32) { 3988 RVT = MVT::v4i32; 3989 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 3990 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 3991 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3992 } else if (V2.getOpcode() != ISD::UNDEF) 3993 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 3994 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3995 if (RVT != VT) 3996 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 3997 return Op; 3998 } 3999 4000 // Binary or unary shufps. 4001 if (X86::isSHUFPMask(PermMask.Val) || 4002 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.Val))) 4003 return Op; 4004 } 4005 4006 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 4007 if (VT == MVT::v8i16) { 4008 SDOperand NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 4009 if (NewOp.Val) 4010 return NewOp; 4011 } 4012 4013 // Handle all 4 wide cases with a number of shuffles except for MMX. 4014 if (NumElems == 4 && !isMMX) 4015 return LowerVECTOR_SHUFFLE_4wide(V1, V2, PermMask, VT, DAG); 4016 4017 return SDOperand(); 4018} 4019 4020SDOperand 4021X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDOperand Op, 4022 SelectionDAG &DAG) { 4023 MVT VT = Op.getValueType(); 4024 if (VT.getSizeInBits() == 8) { 4025 SDOperand Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 4026 Op.getOperand(0), Op.getOperand(1)); 4027 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4028 DAG.getValueType(VT)); 4029 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4030 } else if (VT.getSizeInBits() == 16) { 4031 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 4032 Op.getOperand(0), Op.getOperand(1)); 4033 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4034 DAG.getValueType(VT)); 4035 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4036 } else if (VT == MVT::f32) { 4037 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 4038 // the result back to FR32 register. It's only worth matching if the 4039 // result has a single use which is a store or a bitcast to i32. 4040 if (!Op.hasOneUse()) 4041 return SDOperand(); 4042 SDNode *User = *Op.Val->use_begin(); 4043 if (User->getOpcode() != ISD::STORE && 4044 (User->getOpcode() != ISD::BIT_CONVERT || 4045 User->getValueType(0) != MVT::i32)) 4046 return SDOperand(); 4047 SDOperand Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4048 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 4049 Op.getOperand(1)); 4050 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 4051 } 4052 return SDOperand(); 4053} 4054 4055 4056SDOperand 4057X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 4058 if (!isa<ConstantSDNode>(Op.getOperand(1))) 4059 return SDOperand(); 4060 4061 if (Subtarget->hasSSE41()) { 4062 SDOperand Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 4063 if (Res.Val) 4064 return Res; 4065 } 4066 4067 MVT VT = Op.getValueType(); 4068 // TODO: handle v16i8. 4069 if (VT.getSizeInBits() == 16) { 4070 SDOperand Vec = Op.getOperand(0); 4071 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4072 if (Idx == 0) 4073 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4074 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4075 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4076 Op.getOperand(1))); 4077 // Transform it so it match pextrw which produces a 32-bit result. 4078 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4079 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4080 Op.getOperand(0), Op.getOperand(1)); 4081 SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4082 DAG.getValueType(VT)); 4083 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4084 } else if (VT.getSizeInBits() == 32) { 4085 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4086 if (Idx == 0) 4087 return Op; 4088 // SHUFPS the element to the lowest double word, then movss. 4089 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4090 SmallVector<SDOperand, 8> IdxVec; 4091 IdxVec. 4092 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4093 IdxVec. 4094 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4095 IdxVec. 4096 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4097 IdxVec. 4098 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4099 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4100 &IdxVec[0], IdxVec.size()); 4101 SDOperand Vec = Op.getOperand(0); 4102 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4103 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4104 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4105 DAG.getIntPtrConstant(0)); 4106 } else if (VT.getSizeInBits() == 64) { 4107 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4108 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4109 // to match extract_elt for f64. 4110 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4111 if (Idx == 0) 4112 return Op; 4113 4114 // UNPCKHPD the element to the lowest double word, then movsd. 4115 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4116 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4117 MVT MaskVT = MVT::getIntVectorWithNumElements(2); 4118 SmallVector<SDOperand, 8> IdxVec; 4119 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4120 IdxVec. 4121 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4122 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4123 &IdxVec[0], IdxVec.size()); 4124 SDOperand Vec = Op.getOperand(0); 4125 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4126 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4127 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4128 DAG.getIntPtrConstant(0)); 4129 } 4130 4131 return SDOperand(); 4132} 4133 4134SDOperand 4135X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDOperand Op, SelectionDAG &DAG){ 4136 MVT VT = Op.getValueType(); 4137 MVT EVT = VT.getVectorElementType(); 4138 4139 SDOperand N0 = Op.getOperand(0); 4140 SDOperand N1 = Op.getOperand(1); 4141 SDOperand N2 = Op.getOperand(2); 4142 4143 if ((EVT.getSizeInBits() == 8) || (EVT.getSizeInBits() == 16)) { 4144 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4145 : X86ISD::PINSRW; 4146 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4147 // argument. 4148 if (N1.getValueType() != MVT::i32) 4149 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4150 if (N2.getValueType() != MVT::i32) 4151 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4152 return DAG.getNode(Opc, VT, N0, N1, N2); 4153 } else if (EVT == MVT::f32) { 4154 // Bits [7:6] of the constant are the source select. This will always be 4155 // zero here. The DAG Combiner may combine an extract_elt index into these 4156 // bits. For example (insert (extract, 3), 2) could be matched by putting 4157 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4158 // Bits [5:4] of the constant are the destination select. This is the 4159 // value of the incoming immediate. 4160 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4161 // combine either bitwise AND or insert of float 0.0 to set these bits. 4162 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4163 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4164 } 4165 return SDOperand(); 4166} 4167 4168SDOperand 4169X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 4170 MVT VT = Op.getValueType(); 4171 MVT EVT = VT.getVectorElementType(); 4172 4173 if (Subtarget->hasSSE41()) 4174 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4175 4176 if (EVT == MVT::i8) 4177 return SDOperand(); 4178 4179 SDOperand N0 = Op.getOperand(0); 4180 SDOperand N1 = Op.getOperand(1); 4181 SDOperand N2 = Op.getOperand(2); 4182 4183 if (EVT.getSizeInBits() == 16) { 4184 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4185 // as its second argument. 4186 if (N1.getValueType() != MVT::i32) 4187 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4188 if (N2.getValueType() != MVT::i32) 4189 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4190 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4191 } 4192 return SDOperand(); 4193} 4194 4195SDOperand 4196X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) { 4197 if (Op.getValueType() == MVT::v2f32) 4198 return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f32, 4199 DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i32, 4200 DAG.getNode(ISD::BIT_CONVERT, MVT::i32, 4201 Op.getOperand(0)))); 4202 4203 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4204 MVT VT = MVT::v2i32; 4205 switch (Op.getValueType().getSimpleVT()) { 4206 default: break; 4207 case MVT::v16i8: 4208 case MVT::v8i16: 4209 VT = MVT::v4i32; 4210 break; 4211 } 4212 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4213 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4214} 4215 4216// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4217// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4218// one of the above mentioned nodes. It has to be wrapped because otherwise 4219// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4220// be used to form addressing mode. These wrapped nodes will be selected 4221// into MOV32ri. 4222SDOperand 4223X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) { 4224 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4225 SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(), 4226 getPointerTy(), 4227 CP->getAlignment()); 4228 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4229 // With PIC, the address is actually $g + Offset. 4230 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4231 !Subtarget->isPICStyleRIPRel()) { 4232 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4233 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4234 Result); 4235 } 4236 4237 return Result; 4238} 4239 4240SDOperand 4241X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) { 4242 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4243 SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4244 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4245 // With PIC, the address is actually $g + Offset. 4246 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4247 !Subtarget->isPICStyleRIPRel()) { 4248 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4249 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4250 Result); 4251 } 4252 4253 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4254 // load the value at address GV, not the value of GV itself. This means that 4255 // the GlobalAddress must be in the base or index register of the address, not 4256 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4257 // The same applies for external symbols during PIC codegen 4258 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4259 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4260 PseudoSourceValue::getGOT(), 0); 4261 4262 return Result; 4263} 4264 4265// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4266static SDOperand 4267LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4268 const MVT PtrVT) { 4269 SDOperand InFlag; 4270 SDOperand Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4271 DAG.getNode(X86ISD::GlobalBaseReg, 4272 PtrVT), InFlag); 4273 InFlag = Chain.getValue(1); 4274 4275 // emit leal symbol@TLSGD(,%ebx,1), %eax 4276 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4277 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4278 GA->getValueType(0), 4279 GA->getOffset()); 4280 SDOperand Ops[] = { Chain, TGA, InFlag }; 4281 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4282 InFlag = Result.getValue(2); 4283 Chain = Result.getValue(1); 4284 4285 // call ___tls_get_addr. This function receives its argument in 4286 // the register EAX. 4287 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4288 InFlag = Chain.getValue(1); 4289 4290 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4291 SDOperand Ops1[] = { Chain, 4292 DAG.getTargetExternalSymbol("___tls_get_addr", 4293 PtrVT), 4294 DAG.getRegister(X86::EAX, PtrVT), 4295 DAG.getRegister(X86::EBX, PtrVT), 4296 InFlag }; 4297 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4298 InFlag = Chain.getValue(1); 4299 4300 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4301} 4302 4303// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4304static SDOperand 4305LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4306 const MVT PtrVT) { 4307 SDOperand InFlag, Chain; 4308 4309 // emit leaq symbol@TLSGD(%rip), %rdi 4310 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4311 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4312 GA->getValueType(0), 4313 GA->getOffset()); 4314 SDOperand Ops[] = { DAG.getEntryNode(), TGA}; 4315 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4316 Chain = Result.getValue(1); 4317 InFlag = Result.getValue(2); 4318 4319 // call ___tls_get_addr. This function receives its argument in 4320 // the register RDI. 4321 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4322 InFlag = Chain.getValue(1); 4323 4324 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4325 SDOperand Ops1[] = { Chain, 4326 DAG.getTargetExternalSymbol("___tls_get_addr", 4327 PtrVT), 4328 DAG.getRegister(X86::RDI, PtrVT), 4329 InFlag }; 4330 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4331 InFlag = Chain.getValue(1); 4332 4333 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4334} 4335 4336// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4337// "local exec" model. 4338static SDOperand LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4339 const MVT PtrVT) { 4340 // Get the Thread Pointer 4341 SDOperand ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4342 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4343 // exec) 4344 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4345 GA->getValueType(0), 4346 GA->getOffset()); 4347 SDOperand Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4348 4349 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4350 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4351 PseudoSourceValue::getGOT(), 0); 4352 4353 // The address of the thread local variable is the add of the thread 4354 // pointer with the offset of the variable. 4355 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4356} 4357 4358SDOperand 4359X86TargetLowering::LowerGlobalTLSAddress(SDOperand Op, SelectionDAG &DAG) { 4360 // TODO: implement the "local dynamic" model 4361 // TODO: implement the "initial exec"model for pic executables 4362 assert(Subtarget->isTargetELF() && 4363 "TLS not implemented for non-ELF targets"); 4364 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4365 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4366 // otherwise use the "Local Exec"TLS Model 4367 if (Subtarget->is64Bit()) { 4368 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4369 } else { 4370 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4371 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4372 else 4373 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4374 } 4375} 4376 4377SDOperand 4378X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) { 4379 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4380 SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4381 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4382 // With PIC, the address is actually $g + Offset. 4383 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4384 !Subtarget->isPICStyleRIPRel()) { 4385 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4386 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4387 Result); 4388 } 4389 4390 return Result; 4391} 4392 4393SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) { 4394 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4395 SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4396 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4397 // With PIC, the address is actually $g + Offset. 4398 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4399 !Subtarget->isPICStyleRIPRel()) { 4400 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4401 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4402 Result); 4403 } 4404 4405 return Result; 4406} 4407 4408/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4409/// take a 2 x i32 value to shift plus a shift amount. 4410SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) { 4411 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4412 MVT VT = Op.getValueType(); 4413 unsigned VTBits = VT.getSizeInBits(); 4414 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4415 SDOperand ShOpLo = Op.getOperand(0); 4416 SDOperand ShOpHi = Op.getOperand(1); 4417 SDOperand ShAmt = Op.getOperand(2); 4418 SDOperand Tmp1 = isSRA ? 4419 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4420 DAG.getConstant(0, VT); 4421 4422 SDOperand Tmp2, Tmp3; 4423 if (Op.getOpcode() == ISD::SHL_PARTS) { 4424 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4425 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4426 } else { 4427 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4428 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4429 } 4430 4431 SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4432 DAG.getConstant(VTBits, MVT::i8)); 4433 SDOperand Cond = DAG.getNode(X86ISD::CMP, VT, 4434 AndNode, DAG.getConstant(0, MVT::i8)); 4435 4436 SDOperand Hi, Lo; 4437 SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4438 SDOperand Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 4439 SDOperand Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 4440 4441 if (Op.getOpcode() == ISD::SHL_PARTS) { 4442 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4443 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4444 } else { 4445 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4446 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4447 } 4448 4449 SDOperand Ops[2] = { Lo, Hi }; 4450 return DAG.getMergeValues(Ops, 2); 4451} 4452 4453SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { 4454 MVT SrcVT = Op.getOperand(0).getValueType(); 4455 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4456 "Unknown SINT_TO_FP to lower!"); 4457 4458 // These are really Legal; caller falls through into that case. 4459 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4460 return SDOperand(); 4461 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4462 Subtarget->is64Bit()) 4463 return SDOperand(); 4464 4465 unsigned Size = SrcVT.getSizeInBits()/8; 4466 MachineFunction &MF = DAG.getMachineFunction(); 4467 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4468 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4469 SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4470 StackSlot, 4471 PseudoSourceValue::getFixedStack(SSFI), 0); 4472 4473 // Build the FILD 4474 SDVTList Tys; 4475 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4476 if (useSSE) 4477 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4478 else 4479 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4480 SmallVector<SDOperand, 8> Ops; 4481 Ops.push_back(Chain); 4482 Ops.push_back(StackSlot); 4483 Ops.push_back(DAG.getValueType(SrcVT)); 4484 SDOperand Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4485 Tys, &Ops[0], Ops.size()); 4486 4487 if (useSSE) { 4488 Chain = Result.getValue(1); 4489 SDOperand InFlag = Result.getValue(2); 4490 4491 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4492 // shouldn't be necessary except that RFP cannot be live across 4493 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4494 MachineFunction &MF = DAG.getMachineFunction(); 4495 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4496 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4497 Tys = DAG.getVTList(MVT::Other); 4498 SmallVector<SDOperand, 8> Ops; 4499 Ops.push_back(Chain); 4500 Ops.push_back(Result); 4501 Ops.push_back(StackSlot); 4502 Ops.push_back(DAG.getValueType(Op.getValueType())); 4503 Ops.push_back(InFlag); 4504 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4505 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4506 PseudoSourceValue::getFixedStack(SSFI), 0); 4507 } 4508 4509 return Result; 4510} 4511 4512std::pair<SDOperand,SDOperand> X86TargetLowering:: 4513FP_TO_SINTHelper(SDOperand Op, SelectionDAG &DAG) { 4514 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4515 Op.getValueType().getSimpleVT() >= MVT::i16 && 4516 "Unknown FP_TO_SINT to lower!"); 4517 4518 // These are really Legal. 4519 if (Op.getValueType() == MVT::i32 && 4520 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4521 return std::make_pair(SDOperand(), SDOperand()); 4522 if (Subtarget->is64Bit() && 4523 Op.getValueType() == MVT::i64 && 4524 Op.getOperand(0).getValueType() != MVT::f80) 4525 return std::make_pair(SDOperand(), SDOperand()); 4526 4527 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4528 // stack slot. 4529 MachineFunction &MF = DAG.getMachineFunction(); 4530 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4531 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4532 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4533 unsigned Opc; 4534 switch (Op.getValueType().getSimpleVT()) { 4535 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4536 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4537 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4538 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4539 } 4540 4541 SDOperand Chain = DAG.getEntryNode(); 4542 SDOperand Value = Op.getOperand(0); 4543 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4544 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4545 Chain = DAG.getStore(Chain, Value, StackSlot, 4546 PseudoSourceValue::getFixedStack(SSFI), 0); 4547 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4548 SDOperand Ops[] = { 4549 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4550 }; 4551 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4552 Chain = Value.getValue(1); 4553 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4554 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4555 } 4556 4557 // Build the FP_TO_INT*_IN_MEM 4558 SDOperand Ops[] = { Chain, Value, StackSlot }; 4559 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4560 4561 return std::make_pair(FIST, StackSlot); 4562} 4563 4564SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { 4565 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(Op, DAG); 4566 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4567 if (FIST.Val == 0) return SDOperand(); 4568 4569 // Load the result. 4570 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4571} 4572 4573SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4574 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(SDOperand(N, 0), DAG); 4575 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4576 if (FIST.Val == 0) return 0; 4577 4578 MVT VT = N->getValueType(0); 4579 4580 // Return a load from the stack slot. 4581 SDOperand Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0); 4582 4583 // Use MERGE_VALUES to drop the chain result value and get a node with one 4584 // result. This requires turning off getMergeValues simplification, since 4585 // otherwise it will give us Res back. 4586 return DAG.getMergeValues(&Res, 1, false).Val; 4587} 4588 4589SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) { 4590 MVT VT = Op.getValueType(); 4591 MVT EltVT = VT; 4592 if (VT.isVector()) 4593 EltVT = VT.getVectorElementType(); 4594 std::vector<Constant*> CV; 4595 if (EltVT == MVT::f64) { 4596 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4597 CV.push_back(C); 4598 CV.push_back(C); 4599 } else { 4600 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4601 CV.push_back(C); 4602 CV.push_back(C); 4603 CV.push_back(C); 4604 CV.push_back(C); 4605 } 4606 Constant *C = ConstantVector::get(CV); 4607 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4608 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4609 PseudoSourceValue::getConstantPool(), 0, 4610 false, 16); 4611 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4612} 4613 4614SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) { 4615 MVT VT = Op.getValueType(); 4616 MVT EltVT = VT; 4617 unsigned EltNum = 1; 4618 if (VT.isVector()) { 4619 EltVT = VT.getVectorElementType(); 4620 EltNum = VT.getVectorNumElements(); 4621 } 4622 std::vector<Constant*> CV; 4623 if (EltVT == MVT::f64) { 4624 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4625 CV.push_back(C); 4626 CV.push_back(C); 4627 } else { 4628 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4629 CV.push_back(C); 4630 CV.push_back(C); 4631 CV.push_back(C); 4632 CV.push_back(C); 4633 } 4634 Constant *C = ConstantVector::get(CV); 4635 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4636 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4637 PseudoSourceValue::getConstantPool(), 0, 4638 false, 16); 4639 if (VT.isVector()) { 4640 return DAG.getNode(ISD::BIT_CONVERT, VT, 4641 DAG.getNode(ISD::XOR, MVT::v2i64, 4642 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4643 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4644 } else { 4645 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4646 } 4647} 4648 4649SDOperand X86TargetLowering::LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) { 4650 SDOperand Op0 = Op.getOperand(0); 4651 SDOperand Op1 = Op.getOperand(1); 4652 MVT VT = Op.getValueType(); 4653 MVT SrcVT = Op1.getValueType(); 4654 4655 // If second operand is smaller, extend it first. 4656 if (SrcVT.bitsLT(VT)) { 4657 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4658 SrcVT = VT; 4659 } 4660 // And if it is bigger, shrink it first. 4661 if (SrcVT.bitsGT(VT)) { 4662 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4663 SrcVT = VT; 4664 } 4665 4666 // At this point the operands and the result should have the same 4667 // type, and that won't be f80 since that is not custom lowered. 4668 4669 // First get the sign bit of second operand. 4670 std::vector<Constant*> CV; 4671 if (SrcVT == MVT::f64) { 4672 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4673 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4674 } else { 4675 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4676 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4677 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4678 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4679 } 4680 Constant *C = ConstantVector::get(CV); 4681 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4682 SDOperand Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4683 PseudoSourceValue::getConstantPool(), 0, 4684 false, 16); 4685 SDOperand SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4686 4687 // Shift sign bit right or left if the two operands have different types. 4688 if (SrcVT.bitsGT(VT)) { 4689 // Op0 is MVT::f32, Op1 is MVT::f64. 4690 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4691 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4692 DAG.getConstant(32, MVT::i32)); 4693 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4694 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4695 DAG.getIntPtrConstant(0)); 4696 } 4697 4698 // Clear first operand sign bit. 4699 CV.clear(); 4700 if (VT == MVT::f64) { 4701 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4702 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4703 } else { 4704 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4705 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4706 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4707 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4708 } 4709 C = ConstantVector::get(CV); 4710 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4711 SDOperand Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4712 PseudoSourceValue::getConstantPool(), 0, 4713 false, 16); 4714 SDOperand Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4715 4716 // Or the value with the sign bit. 4717 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4718} 4719 4720SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG) { 4721 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4722 SDOperand Cond; 4723 SDOperand Op0 = Op.getOperand(0); 4724 SDOperand Op1 = Op.getOperand(1); 4725 SDOperand CC = Op.getOperand(2); 4726 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4727 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4728 unsigned X86CC; 4729 4730 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4731 Op0, Op1, DAG)) { 4732 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4733 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4734 DAG.getConstant(X86CC, MVT::i8), Cond); 4735 } 4736 4737 assert(isFP && "Illegal integer SetCC!"); 4738 4739 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4740 switch (SetCCOpcode) { 4741 default: assert(false && "Illegal floating point SetCC!"); 4742 case ISD::SETOEQ: { // !PF & ZF 4743 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4744 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4745 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4746 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4747 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4748 } 4749 case ISD::SETUNE: { // PF | !ZF 4750 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4751 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4752 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4753 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4754 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4755 } 4756 } 4757} 4758 4759SDOperand X86TargetLowering::LowerVSETCC(SDOperand Op, SelectionDAG &DAG) { 4760 SDOperand Cond; 4761 SDOperand Op0 = Op.getOperand(0); 4762 SDOperand Op1 = Op.getOperand(1); 4763 SDOperand CC = Op.getOperand(2); 4764 MVT VT = Op.getValueType(); 4765 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4766 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4767 4768 if (isFP) { 4769 unsigned SSECC = 8; 4770 unsigned Opc = Op0.getValueType() == MVT::v4f32 ? X86ISD::CMPPS : 4771 X86ISD::CMPPD; 4772 bool Swap = false; 4773 4774 switch (SetCCOpcode) { 4775 default: break; 4776 case ISD::SETOEQ: 4777 case ISD::SETEQ: SSECC = 0; break; 4778 case ISD::SETOGT: 4779 case ISD::SETGT: Swap = true; // Fallthrough 4780 case ISD::SETLT: 4781 case ISD::SETOLT: SSECC = 1; break; 4782 case ISD::SETOGE: 4783 case ISD::SETGE: Swap = true; // Fallthrough 4784 case ISD::SETLE: 4785 case ISD::SETOLE: SSECC = 2; break; 4786 case ISD::SETUO: SSECC = 3; break; 4787 case ISD::SETUNE: 4788 case ISD::SETNE: SSECC = 4; break; 4789 case ISD::SETULE: Swap = true; 4790 case ISD::SETUGE: SSECC = 5; break; 4791 case ISD::SETULT: Swap = true; 4792 case ISD::SETUGT: SSECC = 6; break; 4793 case ISD::SETO: SSECC = 7; break; 4794 } 4795 if (Swap) 4796 std::swap(Op0, Op1); 4797 4798 // In the two special cases we can't handle, emit two comparisons. 4799 if (SSECC == 8) { 4800 if (SetCCOpcode == ISD::SETUEQ) { 4801 SDOperand UNORD, EQ; 4802 UNORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 4803 EQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 4804 return DAG.getNode(ISD::OR, VT, UNORD, EQ); 4805 } 4806 else if (SetCCOpcode == ISD::SETONE) { 4807 SDOperand ORD, NEQ; 4808 ORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 4809 NEQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 4810 return DAG.getNode(ISD::AND, VT, ORD, NEQ); 4811 } 4812 assert(0 && "Illegal FP comparison"); 4813 } 4814 // Handle all other FP comparisons here. 4815 return DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 4816 } 4817 4818 // We are handling one of the integer comparisons here. Since SSE only has 4819 // GT and EQ comparisons for integer, swapping operands and multiple 4820 // operations may be required for some comparisons. 4821 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 4822 bool Swap = false, Invert = false, FlipSigns = false; 4823 4824 switch (VT.getSimpleVT()) { 4825 default: break; 4826 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 4827 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 4828 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 4829 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 4830 } 4831 4832 switch (SetCCOpcode) { 4833 default: break; 4834 case ISD::SETNE: Invert = true; 4835 case ISD::SETEQ: Opc = EQOpc; break; 4836 case ISD::SETLT: Swap = true; 4837 case ISD::SETGT: Opc = GTOpc; break; 4838 case ISD::SETGE: Swap = true; 4839 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 4840 case ISD::SETULT: Swap = true; 4841 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 4842 case ISD::SETUGE: Swap = true; 4843 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 4844 } 4845 if (Swap) 4846 std::swap(Op0, Op1); 4847 4848 // Since SSE has no unsigned integer comparisons, we need to flip the sign 4849 // bits of the inputs before performing those operations. 4850 if (FlipSigns) { 4851 MVT EltVT = VT.getVectorElementType(); 4852 SDOperand SignBit = DAG.getConstant(EltVT.getIntegerVTSignBit(), EltVT); 4853 std::vector<SDOperand> SignBits(VT.getVectorNumElements(), SignBit); 4854 SDOperand SignVec = DAG.getNode(ISD::BUILD_VECTOR, VT, &SignBits[0], 4855 SignBits.size()); 4856 Op0 = DAG.getNode(ISD::XOR, VT, Op0, SignVec); 4857 Op1 = DAG.getNode(ISD::XOR, VT, Op1, SignVec); 4858 } 4859 4860 SDOperand Result = DAG.getNode(Opc, VT, Op0, Op1); 4861 4862 // If the logical-not of the result is required, perform that now. 4863 if (Invert) { 4864 MVT EltVT = VT.getVectorElementType(); 4865 SDOperand NegOne = DAG.getConstant(EltVT.getIntegerVTBitMask(), EltVT); 4866 std::vector<SDOperand> NegOnes(VT.getVectorNumElements(), NegOne); 4867 SDOperand NegOneV = DAG.getNode(ISD::BUILD_VECTOR, VT, &NegOnes[0], 4868 NegOnes.size()); 4869 Result = DAG.getNode(ISD::XOR, VT, Result, NegOneV); 4870 } 4871 return Result; 4872} 4873 4874SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) { 4875 bool addTest = true; 4876 SDOperand Cond = Op.getOperand(0); 4877 SDOperand CC; 4878 4879 if (Cond.getOpcode() == ISD::SETCC) 4880 Cond = LowerSETCC(Cond, DAG); 4881 4882 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4883 // setting operand in place of the X86ISD::SETCC. 4884 if (Cond.getOpcode() == X86ISD::SETCC) { 4885 CC = Cond.getOperand(0); 4886 4887 SDOperand Cmp = Cond.getOperand(1); 4888 unsigned Opc = Cmp.getOpcode(); 4889 MVT VT = Op.getValueType(); 4890 4891 bool IllegalFPCMov = false; 4892 if (VT.isFloatingPoint() && !VT.isVector() && 4893 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4894 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4895 4896 if ((Opc == X86ISD::CMP || 4897 Opc == X86ISD::COMI || 4898 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4899 Cond = Cmp; 4900 addTest = false; 4901 } 4902 } 4903 4904 if (addTest) { 4905 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4906 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4907 } 4908 4909 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4910 MVT::Flag); 4911 SmallVector<SDOperand, 4> Ops; 4912 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4913 // condition is true. 4914 Ops.push_back(Op.getOperand(2)); 4915 Ops.push_back(Op.getOperand(1)); 4916 Ops.push_back(CC); 4917 Ops.push_back(Cond); 4918 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4919} 4920 4921SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) { 4922 bool addTest = true; 4923 SDOperand Chain = Op.getOperand(0); 4924 SDOperand Cond = Op.getOperand(1); 4925 SDOperand Dest = Op.getOperand(2); 4926 SDOperand CC; 4927 4928 if (Cond.getOpcode() == ISD::SETCC) 4929 Cond = LowerSETCC(Cond, DAG); 4930 4931 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4932 // setting operand in place of the X86ISD::SETCC. 4933 if (Cond.getOpcode() == X86ISD::SETCC) { 4934 CC = Cond.getOperand(0); 4935 4936 SDOperand Cmp = Cond.getOperand(1); 4937 unsigned Opc = Cmp.getOpcode(); 4938 if (Opc == X86ISD::CMP || 4939 Opc == X86ISD::COMI || 4940 Opc == X86ISD::UCOMI) { 4941 Cond = Cmp; 4942 addTest = false; 4943 } 4944 } 4945 4946 if (addTest) { 4947 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4948 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4949 } 4950 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 4951 Chain, Op.getOperand(2), CC, Cond); 4952} 4953 4954 4955// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 4956// Calls to _alloca is needed to probe the stack when allocating more than 4k 4957// bytes in one go. Touching the stack at 4K increments is necessary to ensure 4958// that the guard pages used by the OS virtual memory manager are allocated in 4959// correct sequence. 4960SDOperand 4961X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op, 4962 SelectionDAG &DAG) { 4963 assert(Subtarget->isTargetCygMing() && 4964 "This should be used only on Cygwin/Mingw targets"); 4965 4966 // Get the inputs. 4967 SDOperand Chain = Op.getOperand(0); 4968 SDOperand Size = Op.getOperand(1); 4969 // FIXME: Ensure alignment here 4970 4971 SDOperand Flag; 4972 4973 MVT IntPtr = getPointerTy(); 4974 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 4975 4976 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 4977 4978 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 4979 Flag = Chain.getValue(1); 4980 4981 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4982 SDOperand Ops[] = { Chain, 4983 DAG.getTargetExternalSymbol("_alloca", IntPtr), 4984 DAG.getRegister(X86::EAX, IntPtr), 4985 DAG.getRegister(X86StackPtr, SPTy), 4986 Flag }; 4987 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 4988 Flag = Chain.getValue(1); 4989 4990 Chain = DAG.getCALLSEQ_END(Chain, 4991 DAG.getIntPtrConstant(0), 4992 DAG.getIntPtrConstant(0), 4993 Flag); 4994 4995 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 4996 4997 SDOperand Ops1[2] = { Chain.getValue(0), Chain }; 4998 return DAG.getMergeValues(Ops1, 2); 4999} 5000 5001SDOperand 5002X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 5003 SDOperand Chain, 5004 SDOperand Dst, SDOperand Src, 5005 SDOperand Size, unsigned Align, 5006 const Value *DstSV, uint64_t DstSVOff) { 5007 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5008 5009 /// If not DWORD aligned or size is more than the threshold, call the library. 5010 /// The libc version is likely to be faster for these cases. It can use the 5011 /// address value and run time information about the CPU. 5012 if ((Align & 3) == 0 || 5013 !ConstantSize || 5014 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 5015 SDOperand InFlag(0, 0); 5016 5017 // Check to see if there is a specialized entry-point for memory zeroing. 5018 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 5019 if (const char *bzeroEntry = 5020 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 5021 MVT IntPtr = getPointerTy(); 5022 const Type *IntPtrTy = getTargetData()->getIntPtrType(); 5023 TargetLowering::ArgListTy Args; 5024 TargetLowering::ArgListEntry Entry; 5025 Entry.Node = Dst; 5026 Entry.Ty = IntPtrTy; 5027 Args.push_back(Entry); 5028 Entry.Node = Size; 5029 Args.push_back(Entry); 5030 std::pair<SDOperand,SDOperand> CallResult = 5031 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 5032 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 5033 Args, DAG); 5034 return CallResult.second; 5035 } 5036 5037 // Otherwise have the target-independent code call memset. 5038 return SDOperand(); 5039 } 5040 5041 uint64_t SizeVal = ConstantSize->getValue(); 5042 SDOperand InFlag(0, 0); 5043 MVT AVT; 5044 SDOperand Count; 5045 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 5046 unsigned BytesLeft = 0; 5047 bool TwoRepStos = false; 5048 if (ValC) { 5049 unsigned ValReg; 5050 uint64_t Val = ValC->getValue() & 255; 5051 5052 // If the value is a constant, then we can potentially use larger sets. 5053 switch (Align & 3) { 5054 case 2: // WORD aligned 5055 AVT = MVT::i16; 5056 ValReg = X86::AX; 5057 Val = (Val << 8) | Val; 5058 break; 5059 case 0: // DWORD aligned 5060 AVT = MVT::i32; 5061 ValReg = X86::EAX; 5062 Val = (Val << 8) | Val; 5063 Val = (Val << 16) | Val; 5064 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 5065 AVT = MVT::i64; 5066 ValReg = X86::RAX; 5067 Val = (Val << 32) | Val; 5068 } 5069 break; 5070 default: // Byte aligned 5071 AVT = MVT::i8; 5072 ValReg = X86::AL; 5073 Count = DAG.getIntPtrConstant(SizeVal); 5074 break; 5075 } 5076 5077 if (AVT.bitsGT(MVT::i8)) { 5078 unsigned UBytes = AVT.getSizeInBits() / 8; 5079 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 5080 BytesLeft = SizeVal % UBytes; 5081 } 5082 5083 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 5084 InFlag); 5085 InFlag = Chain.getValue(1); 5086 } else { 5087 AVT = MVT::i8; 5088 Count = DAG.getIntPtrConstant(SizeVal); 5089 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 5090 InFlag = Chain.getValue(1); 5091 } 5092 5093 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5094 Count, InFlag); 5095 InFlag = Chain.getValue(1); 5096 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5097 Dst, InFlag); 5098 InFlag = Chain.getValue(1); 5099 5100 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5101 SmallVector<SDOperand, 8> Ops; 5102 Ops.push_back(Chain); 5103 Ops.push_back(DAG.getValueType(AVT)); 5104 Ops.push_back(InFlag); 5105 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5106 5107 if (TwoRepStos) { 5108 InFlag = Chain.getValue(1); 5109 Count = Size; 5110 MVT CVT = Count.getValueType(); 5111 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count, 5112 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 5113 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 5114 Left, InFlag); 5115 InFlag = Chain.getValue(1); 5116 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5117 Ops.clear(); 5118 Ops.push_back(Chain); 5119 Ops.push_back(DAG.getValueType(MVT::i8)); 5120 Ops.push_back(InFlag); 5121 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5122 } else if (BytesLeft) { 5123 // Handle the last 1 - 7 bytes. 5124 unsigned Offset = SizeVal - BytesLeft; 5125 MVT AddrVT = Dst.getValueType(); 5126 MVT SizeVT = Size.getValueType(); 5127 5128 Chain = DAG.getMemset(Chain, 5129 DAG.getNode(ISD::ADD, AddrVT, Dst, 5130 DAG.getConstant(Offset, AddrVT)), 5131 Src, 5132 DAG.getConstant(BytesLeft, SizeVT), 5133 Align, DstSV, DstSVOff + Offset); 5134 } 5135 5136 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 5137 return Chain; 5138} 5139 5140SDOperand 5141X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 5142 SDOperand Chain, 5143 SDOperand Dst, SDOperand Src, 5144 SDOperand Size, unsigned Align, 5145 bool AlwaysInline, 5146 const Value *DstSV, uint64_t DstSVOff, 5147 const Value *SrcSV, uint64_t SrcSVOff){ 5148 5149 // This requires the copy size to be a constant, preferrably 5150 // within a subtarget-specific limit. 5151 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5152 if (!ConstantSize) 5153 return SDOperand(); 5154 uint64_t SizeVal = ConstantSize->getValue(); 5155 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 5156 return SDOperand(); 5157 5158 MVT AVT; 5159 unsigned BytesLeft = 0; 5160 if (Align >= 8 && Subtarget->is64Bit()) 5161 AVT = MVT::i64; 5162 else if (Align >= 4) 5163 AVT = MVT::i32; 5164 else if (Align >= 2) 5165 AVT = MVT::i16; 5166 else 5167 AVT = MVT::i8; 5168 5169 unsigned UBytes = AVT.getSizeInBits() / 8; 5170 unsigned CountVal = SizeVal / UBytes; 5171 SDOperand Count = DAG.getIntPtrConstant(CountVal); 5172 BytesLeft = SizeVal % UBytes; 5173 5174 SDOperand InFlag(0, 0); 5175 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5176 Count, InFlag); 5177 InFlag = Chain.getValue(1); 5178 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5179 Dst, InFlag); 5180 InFlag = Chain.getValue(1); 5181 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 5182 Src, InFlag); 5183 InFlag = Chain.getValue(1); 5184 5185 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5186 SmallVector<SDOperand, 8> Ops; 5187 Ops.push_back(Chain); 5188 Ops.push_back(DAG.getValueType(AVT)); 5189 Ops.push_back(InFlag); 5190 SDOperand RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5191 5192 SmallVector<SDOperand, 4> Results; 5193 Results.push_back(RepMovs); 5194 if (BytesLeft) { 5195 // Handle the last 1 - 7 bytes. 5196 unsigned Offset = SizeVal - BytesLeft; 5197 MVT DstVT = Dst.getValueType(); 5198 MVT SrcVT = Src.getValueType(); 5199 MVT SizeVT = Size.getValueType(); 5200 Results.push_back(DAG.getMemcpy(Chain, 5201 DAG.getNode(ISD::ADD, DstVT, Dst, 5202 DAG.getConstant(Offset, DstVT)), 5203 DAG.getNode(ISD::ADD, SrcVT, Src, 5204 DAG.getConstant(Offset, SrcVT)), 5205 DAG.getConstant(BytesLeft, SizeVT), 5206 Align, AlwaysInline, 5207 DstSV, DstSVOff + Offset, 5208 SrcSV, SrcSVOff + Offset)); 5209 } 5210 5211 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5212} 5213 5214/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5215SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5216 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5217 SDOperand TheChain = N->getOperand(0); 5218 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5219 if (Subtarget->is64Bit()) { 5220 SDOperand rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5221 SDOperand rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5222 MVT::i64, rax.getValue(2)); 5223 SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5224 DAG.getConstant(32, MVT::i8)); 5225 SDOperand Ops[] = { 5226 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5227 }; 5228 5229 return DAG.getMergeValues(Ops, 2).Val; 5230 } 5231 5232 SDOperand eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5233 SDOperand edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5234 MVT::i32, eax.getValue(2)); 5235 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5236 SDOperand Ops[] = { eax, edx }; 5237 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5238 5239 // Use a MERGE_VALUES to return the value and chain. 5240 Ops[1] = edx.getValue(1); 5241 return DAG.getMergeValues(Ops, 2).Val; 5242} 5243 5244SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) { 5245 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5246 5247 if (!Subtarget->is64Bit()) { 5248 // vastart just stores the address of the VarArgsFrameIndex slot into the 5249 // memory location argument. 5250 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5251 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5252 } 5253 5254 // __va_list_tag: 5255 // gp_offset (0 - 6 * 8) 5256 // fp_offset (48 - 48 + 8 * 16) 5257 // overflow_arg_area (point to parameters coming in memory). 5258 // reg_save_area 5259 SmallVector<SDOperand, 8> MemOps; 5260 SDOperand FIN = Op.getOperand(1); 5261 // Store gp_offset 5262 SDOperand Store = DAG.getStore(Op.getOperand(0), 5263 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5264 FIN, SV, 0); 5265 MemOps.push_back(Store); 5266 5267 // Store fp_offset 5268 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5269 Store = DAG.getStore(Op.getOperand(0), 5270 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5271 FIN, SV, 0); 5272 MemOps.push_back(Store); 5273 5274 // Store ptr to overflow_arg_area 5275 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5276 SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5277 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5278 MemOps.push_back(Store); 5279 5280 // Store ptr to reg_save_area. 5281 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5282 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5283 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5284 MemOps.push_back(Store); 5285 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5286} 5287 5288SDOperand X86TargetLowering::LowerVAARG(SDOperand Op, SelectionDAG &DAG) { 5289 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5290 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5291 SDOperand Chain = Op.getOperand(0); 5292 SDOperand SrcPtr = Op.getOperand(1); 5293 SDOperand SrcSV = Op.getOperand(2); 5294 5295 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5296 abort(); 5297 return SDOperand(); 5298} 5299 5300SDOperand X86TargetLowering::LowerVACOPY(SDOperand Op, SelectionDAG &DAG) { 5301 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5302 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5303 SDOperand Chain = Op.getOperand(0); 5304 SDOperand DstPtr = Op.getOperand(1); 5305 SDOperand SrcPtr = Op.getOperand(2); 5306 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5307 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5308 5309 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5310 DAG.getIntPtrConstant(24), 8, false, 5311 DstSV, 0, SrcSV, 0); 5312} 5313 5314SDOperand 5315X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) { 5316 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5317 switch (IntNo) { 5318 default: return SDOperand(); // Don't custom lower most intrinsics. 5319 // Comparison intrinsics. 5320 case Intrinsic::x86_sse_comieq_ss: 5321 case Intrinsic::x86_sse_comilt_ss: 5322 case Intrinsic::x86_sse_comile_ss: 5323 case Intrinsic::x86_sse_comigt_ss: 5324 case Intrinsic::x86_sse_comige_ss: 5325 case Intrinsic::x86_sse_comineq_ss: 5326 case Intrinsic::x86_sse_ucomieq_ss: 5327 case Intrinsic::x86_sse_ucomilt_ss: 5328 case Intrinsic::x86_sse_ucomile_ss: 5329 case Intrinsic::x86_sse_ucomigt_ss: 5330 case Intrinsic::x86_sse_ucomige_ss: 5331 case Intrinsic::x86_sse_ucomineq_ss: 5332 case Intrinsic::x86_sse2_comieq_sd: 5333 case Intrinsic::x86_sse2_comilt_sd: 5334 case Intrinsic::x86_sse2_comile_sd: 5335 case Intrinsic::x86_sse2_comigt_sd: 5336 case Intrinsic::x86_sse2_comige_sd: 5337 case Intrinsic::x86_sse2_comineq_sd: 5338 case Intrinsic::x86_sse2_ucomieq_sd: 5339 case Intrinsic::x86_sse2_ucomilt_sd: 5340 case Intrinsic::x86_sse2_ucomile_sd: 5341 case Intrinsic::x86_sse2_ucomigt_sd: 5342 case Intrinsic::x86_sse2_ucomige_sd: 5343 case Intrinsic::x86_sse2_ucomineq_sd: { 5344 unsigned Opc = 0; 5345 ISD::CondCode CC = ISD::SETCC_INVALID; 5346 switch (IntNo) { 5347 default: break; 5348 case Intrinsic::x86_sse_comieq_ss: 5349 case Intrinsic::x86_sse2_comieq_sd: 5350 Opc = X86ISD::COMI; 5351 CC = ISD::SETEQ; 5352 break; 5353 case Intrinsic::x86_sse_comilt_ss: 5354 case Intrinsic::x86_sse2_comilt_sd: 5355 Opc = X86ISD::COMI; 5356 CC = ISD::SETLT; 5357 break; 5358 case Intrinsic::x86_sse_comile_ss: 5359 case Intrinsic::x86_sse2_comile_sd: 5360 Opc = X86ISD::COMI; 5361 CC = ISD::SETLE; 5362 break; 5363 case Intrinsic::x86_sse_comigt_ss: 5364 case Intrinsic::x86_sse2_comigt_sd: 5365 Opc = X86ISD::COMI; 5366 CC = ISD::SETGT; 5367 break; 5368 case Intrinsic::x86_sse_comige_ss: 5369 case Intrinsic::x86_sse2_comige_sd: 5370 Opc = X86ISD::COMI; 5371 CC = ISD::SETGE; 5372 break; 5373 case Intrinsic::x86_sse_comineq_ss: 5374 case Intrinsic::x86_sse2_comineq_sd: 5375 Opc = X86ISD::COMI; 5376 CC = ISD::SETNE; 5377 break; 5378 case Intrinsic::x86_sse_ucomieq_ss: 5379 case Intrinsic::x86_sse2_ucomieq_sd: 5380 Opc = X86ISD::UCOMI; 5381 CC = ISD::SETEQ; 5382 break; 5383 case Intrinsic::x86_sse_ucomilt_ss: 5384 case Intrinsic::x86_sse2_ucomilt_sd: 5385 Opc = X86ISD::UCOMI; 5386 CC = ISD::SETLT; 5387 break; 5388 case Intrinsic::x86_sse_ucomile_ss: 5389 case Intrinsic::x86_sse2_ucomile_sd: 5390 Opc = X86ISD::UCOMI; 5391 CC = ISD::SETLE; 5392 break; 5393 case Intrinsic::x86_sse_ucomigt_ss: 5394 case Intrinsic::x86_sse2_ucomigt_sd: 5395 Opc = X86ISD::UCOMI; 5396 CC = ISD::SETGT; 5397 break; 5398 case Intrinsic::x86_sse_ucomige_ss: 5399 case Intrinsic::x86_sse2_ucomige_sd: 5400 Opc = X86ISD::UCOMI; 5401 CC = ISD::SETGE; 5402 break; 5403 case Intrinsic::x86_sse_ucomineq_ss: 5404 case Intrinsic::x86_sse2_ucomineq_sd: 5405 Opc = X86ISD::UCOMI; 5406 CC = ISD::SETNE; 5407 break; 5408 } 5409 5410 unsigned X86CC; 5411 SDOperand LHS = Op.getOperand(1); 5412 SDOperand RHS = Op.getOperand(2); 5413 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5414 5415 SDOperand Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5416 SDOperand SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5417 DAG.getConstant(X86CC, MVT::i8), Cond); 5418 return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC); 5419 } 5420 5421 // Fix vector shift instructions where the last operand is a non-immediate 5422 // i32 value. 5423 case Intrinsic::x86_sse2_pslli_w: 5424 case Intrinsic::x86_sse2_pslli_d: 5425 case Intrinsic::x86_sse2_pslli_q: 5426 case Intrinsic::x86_sse2_psrli_w: 5427 case Intrinsic::x86_sse2_psrli_d: 5428 case Intrinsic::x86_sse2_psrli_q: 5429 case Intrinsic::x86_sse2_psrai_w: 5430 case Intrinsic::x86_sse2_psrai_d: 5431 case Intrinsic::x86_mmx_pslli_w: 5432 case Intrinsic::x86_mmx_pslli_d: 5433 case Intrinsic::x86_mmx_pslli_q: 5434 case Intrinsic::x86_mmx_psrli_w: 5435 case Intrinsic::x86_mmx_psrli_d: 5436 case Intrinsic::x86_mmx_psrli_q: 5437 case Intrinsic::x86_mmx_psrai_w: 5438 case Intrinsic::x86_mmx_psrai_d: { 5439 SDOperand ShAmt = Op.getOperand(2); 5440 if (isa<ConstantSDNode>(ShAmt)) 5441 return SDOperand(); 5442 5443 unsigned NewIntNo = 0; 5444 MVT ShAmtVT = MVT::v4i32; 5445 switch (IntNo) { 5446 case Intrinsic::x86_sse2_pslli_w: 5447 NewIntNo = Intrinsic::x86_sse2_psll_w; 5448 break; 5449 case Intrinsic::x86_sse2_pslli_d: 5450 NewIntNo = Intrinsic::x86_sse2_psll_d; 5451 break; 5452 case Intrinsic::x86_sse2_pslli_q: 5453 NewIntNo = Intrinsic::x86_sse2_psll_q; 5454 break; 5455 case Intrinsic::x86_sse2_psrli_w: 5456 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5457 break; 5458 case Intrinsic::x86_sse2_psrli_d: 5459 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5460 break; 5461 case Intrinsic::x86_sse2_psrli_q: 5462 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5463 break; 5464 case Intrinsic::x86_sse2_psrai_w: 5465 NewIntNo = Intrinsic::x86_sse2_psra_w; 5466 break; 5467 case Intrinsic::x86_sse2_psrai_d: 5468 NewIntNo = Intrinsic::x86_sse2_psra_d; 5469 break; 5470 default: { 5471 ShAmtVT = MVT::v2i32; 5472 switch (IntNo) { 5473 case Intrinsic::x86_mmx_pslli_w: 5474 NewIntNo = Intrinsic::x86_mmx_psll_w; 5475 break; 5476 case Intrinsic::x86_mmx_pslli_d: 5477 NewIntNo = Intrinsic::x86_mmx_psll_d; 5478 break; 5479 case Intrinsic::x86_mmx_pslli_q: 5480 NewIntNo = Intrinsic::x86_mmx_psll_q; 5481 break; 5482 case Intrinsic::x86_mmx_psrli_w: 5483 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5484 break; 5485 case Intrinsic::x86_mmx_psrli_d: 5486 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5487 break; 5488 case Intrinsic::x86_mmx_psrli_q: 5489 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5490 break; 5491 case Intrinsic::x86_mmx_psrai_w: 5492 NewIntNo = Intrinsic::x86_mmx_psra_w; 5493 break; 5494 case Intrinsic::x86_mmx_psrai_d: 5495 NewIntNo = Intrinsic::x86_mmx_psra_d; 5496 break; 5497 default: abort(); // Can't reach here. 5498 } 5499 break; 5500 } 5501 } 5502 MVT VT = Op.getValueType(); 5503 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5504 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5505 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5506 DAG.getConstant(NewIntNo, MVT::i32), 5507 Op.getOperand(1), ShAmt); 5508 } 5509 } 5510} 5511 5512SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) { 5513 // Depths > 0 not supported yet! 5514 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5515 return SDOperand(); 5516 5517 // Just load the return address 5518 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5519 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5520} 5521 5522SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) { 5523 // Depths > 0 not supported yet! 5524 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5525 return SDOperand(); 5526 5527 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5528 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5529 DAG.getIntPtrConstant(!Subtarget->is64Bit() ? 4 : 8)); 5530} 5531 5532SDOperand X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDOperand Op, 5533 SelectionDAG &DAG) { 5534 // Is not yet supported on x86-64 5535 if (Subtarget->is64Bit()) 5536 return SDOperand(); 5537 5538 return DAG.getIntPtrConstant(8); 5539} 5540 5541SDOperand X86TargetLowering::LowerEH_RETURN(SDOperand Op, SelectionDAG &DAG) 5542{ 5543 assert(!Subtarget->is64Bit() && 5544 "Lowering of eh_return builtin is not supported yet on x86-64"); 5545 5546 MachineFunction &MF = DAG.getMachineFunction(); 5547 SDOperand Chain = Op.getOperand(0); 5548 SDOperand Offset = Op.getOperand(1); 5549 SDOperand Handler = Op.getOperand(2); 5550 5551 SDOperand Frame = DAG.getRegister(RegInfo->getFrameRegister(MF), 5552 getPointerTy()); 5553 5554 SDOperand StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5555 DAG.getIntPtrConstant(-4UL)); 5556 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5557 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5558 Chain = DAG.getCopyToReg(Chain, X86::ECX, StoreAddr); 5559 MF.getRegInfo().addLiveOut(X86::ECX); 5560 5561 return DAG.getNode(X86ISD::EH_RETURN, MVT::Other, 5562 Chain, DAG.getRegister(X86::ECX, getPointerTy())); 5563} 5564 5565SDOperand X86TargetLowering::LowerTRAMPOLINE(SDOperand Op, 5566 SelectionDAG &DAG) { 5567 SDOperand Root = Op.getOperand(0); 5568 SDOperand Trmp = Op.getOperand(1); // trampoline 5569 SDOperand FPtr = Op.getOperand(2); // nested function 5570 SDOperand Nest = Op.getOperand(3); // 'nest' parameter value 5571 5572 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5573 5574 const X86InstrInfo *TII = 5575 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5576 5577 if (Subtarget->is64Bit()) { 5578 SDOperand OutChains[6]; 5579 5580 // Large code-model. 5581 5582 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5583 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5584 5585 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5586 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5587 5588 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5589 5590 // Load the pointer to the nested function into R11. 5591 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5592 SDOperand Addr = Trmp; 5593 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5594 TrmpAddr, 0); 5595 5596 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5597 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5598 5599 // Load the 'nest' parameter value into R10. 5600 // R10 is specified in X86CallingConv.td 5601 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5602 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5603 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5604 TrmpAddr, 10); 5605 5606 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5607 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5608 5609 // Jump to the nested function. 5610 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5611 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5612 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5613 TrmpAddr, 20); 5614 5615 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5616 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5617 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5618 TrmpAddr, 22); 5619 5620 SDOperand Ops[] = 5621 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5622 return DAG.getMergeValues(Ops, 2); 5623 } else { 5624 const Function *Func = 5625 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5626 unsigned CC = Func->getCallingConv(); 5627 unsigned NestReg; 5628 5629 switch (CC) { 5630 default: 5631 assert(0 && "Unsupported calling convention"); 5632 case CallingConv::C: 5633 case CallingConv::X86_StdCall: { 5634 // Pass 'nest' parameter in ECX. 5635 // Must be kept in sync with X86CallingConv.td 5636 NestReg = X86::ECX; 5637 5638 // Check that ECX wasn't needed by an 'inreg' parameter. 5639 const FunctionType *FTy = Func->getFunctionType(); 5640 const PAListPtr &Attrs = Func->getParamAttrs(); 5641 5642 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5643 unsigned InRegCount = 0; 5644 unsigned Idx = 1; 5645 5646 for (FunctionType::param_iterator I = FTy->param_begin(), 5647 E = FTy->param_end(); I != E; ++I, ++Idx) 5648 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5649 // FIXME: should only count parameters that are lowered to integers. 5650 InRegCount += (getTargetData()->getTypeSizeInBits(*I) + 31) / 32; 5651 5652 if (InRegCount > 2) { 5653 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5654 abort(); 5655 } 5656 } 5657 break; 5658 } 5659 case CallingConv::X86_FastCall: 5660 // Pass 'nest' parameter in EAX. 5661 // Must be kept in sync with X86CallingConv.td 5662 NestReg = X86::EAX; 5663 break; 5664 } 5665 5666 SDOperand OutChains[4]; 5667 SDOperand Addr, Disp; 5668 5669 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5670 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5671 5672 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5673 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5674 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5675 Trmp, TrmpAddr, 0); 5676 5677 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5678 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5679 5680 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5681 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5682 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5683 TrmpAddr, 5, false, 1); 5684 5685 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5686 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5687 5688 SDOperand Ops[] = 5689 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5690 return DAG.getMergeValues(Ops, 2); 5691 } 5692} 5693 5694SDOperand X86TargetLowering::LowerFLT_ROUNDS_(SDOperand Op, SelectionDAG &DAG) { 5695 /* 5696 The rounding mode is in bits 11:10 of FPSR, and has the following 5697 settings: 5698 00 Round to nearest 5699 01 Round to -inf 5700 10 Round to +inf 5701 11 Round to 0 5702 5703 FLT_ROUNDS, on the other hand, expects the following: 5704 -1 Undefined 5705 0 Round to 0 5706 1 Round to nearest 5707 2 Round to +inf 5708 3 Round to -inf 5709 5710 To perform the conversion, we do: 5711 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5712 */ 5713 5714 MachineFunction &MF = DAG.getMachineFunction(); 5715 const TargetMachine &TM = MF.getTarget(); 5716 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5717 unsigned StackAlignment = TFI.getStackAlignment(); 5718 MVT VT = Op.getValueType(); 5719 5720 // Save FP Control Word to stack slot 5721 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5722 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5723 5724 SDOperand Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5725 DAG.getEntryNode(), StackSlot); 5726 5727 // Load FP Control Word from stack slot 5728 SDOperand CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5729 5730 // Transform as necessary 5731 SDOperand CWD1 = 5732 DAG.getNode(ISD::SRL, MVT::i16, 5733 DAG.getNode(ISD::AND, MVT::i16, 5734 CWD, DAG.getConstant(0x800, MVT::i16)), 5735 DAG.getConstant(11, MVT::i8)); 5736 SDOperand CWD2 = 5737 DAG.getNode(ISD::SRL, MVT::i16, 5738 DAG.getNode(ISD::AND, MVT::i16, 5739 CWD, DAG.getConstant(0x400, MVT::i16)), 5740 DAG.getConstant(9, MVT::i8)); 5741 5742 SDOperand RetVal = 5743 DAG.getNode(ISD::AND, MVT::i16, 5744 DAG.getNode(ISD::ADD, MVT::i16, 5745 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5746 DAG.getConstant(1, MVT::i16)), 5747 DAG.getConstant(3, MVT::i16)); 5748 5749 5750 return DAG.getNode((VT.getSizeInBits() < 16 ? 5751 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5752} 5753 5754SDOperand X86TargetLowering::LowerCTLZ(SDOperand Op, SelectionDAG &DAG) { 5755 MVT VT = Op.getValueType(); 5756 MVT OpVT = VT; 5757 unsigned NumBits = VT.getSizeInBits(); 5758 5759 Op = Op.getOperand(0); 5760 if (VT == MVT::i8) { 5761 // Zero extend to i32 since there is not an i8 bsr. 5762 OpVT = MVT::i32; 5763 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5764 } 5765 5766 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5767 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5768 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5769 5770 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5771 SmallVector<SDOperand, 4> Ops; 5772 Ops.push_back(Op); 5773 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5774 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5775 Ops.push_back(Op.getValue(1)); 5776 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5777 5778 // Finally xor with NumBits-1. 5779 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5780 5781 if (VT == MVT::i8) 5782 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5783 return Op; 5784} 5785 5786SDOperand X86TargetLowering::LowerCTTZ(SDOperand Op, SelectionDAG &DAG) { 5787 MVT VT = Op.getValueType(); 5788 MVT OpVT = VT; 5789 unsigned NumBits = VT.getSizeInBits(); 5790 5791 Op = Op.getOperand(0); 5792 if (VT == MVT::i8) { 5793 OpVT = MVT::i32; 5794 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5795 } 5796 5797 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5798 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5799 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5800 5801 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5802 SmallVector<SDOperand, 4> Ops; 5803 Ops.push_back(Op); 5804 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5805 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5806 Ops.push_back(Op.getValue(1)); 5807 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5808 5809 if (VT == MVT::i8) 5810 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5811 return Op; 5812} 5813 5814SDOperand X86TargetLowering::LowerCMP_SWAP(SDOperand Op, SelectionDAG &DAG) { 5815 MVT T = Op.getValueType(); 5816 unsigned Reg = 0; 5817 unsigned size = 0; 5818 switch(T.getSimpleVT()) { 5819 default: 5820 assert(false && "Invalid value type!"); 5821 case MVT::i8: Reg = X86::AL; size = 1; break; 5822 case MVT::i16: Reg = X86::AX; size = 2; break; 5823 case MVT::i32: Reg = X86::EAX; size = 4; break; 5824 case MVT::i64: 5825 if (Subtarget->is64Bit()) { 5826 Reg = X86::RAX; size = 8; 5827 } else //Should go away when LowerType stuff lands 5828 return SDOperand(ExpandATOMIC_CMP_SWAP(Op.Val, DAG), 0); 5829 break; 5830 }; 5831 SDOperand cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5832 Op.getOperand(3), SDOperand()); 5833 SDOperand Ops[] = { cpIn.getValue(0), 5834 Op.getOperand(1), 5835 Op.getOperand(2), 5836 DAG.getTargetConstant(size, MVT::i8), 5837 cpIn.getValue(1) }; 5838 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5839 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5840 SDOperand cpOut = 5841 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5842 return cpOut; 5843} 5844 5845SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, SelectionDAG &DAG) { 5846 MVT T = Op->getValueType(0); 5847 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5848 SDOperand cpInL, cpInH; 5849 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5850 DAG.getConstant(0, MVT::i32)); 5851 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5852 DAG.getConstant(1, MVT::i32)); 5853 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5854 cpInL, SDOperand()); 5855 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5856 cpInH, cpInL.getValue(1)); 5857 SDOperand swapInL, swapInH; 5858 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5859 DAG.getConstant(0, MVT::i32)); 5860 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5861 DAG.getConstant(1, MVT::i32)); 5862 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5863 swapInL, cpInH.getValue(1)); 5864 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5865 swapInH, swapInL.getValue(1)); 5866 SDOperand Ops[] = { swapInH.getValue(0), 5867 Op->getOperand(1), 5868 swapInH.getValue(1)}; 5869 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5870 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5871 SDOperand cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5872 Result.getValue(1)); 5873 SDOperand cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5874 cpOutL.getValue(2)); 5875 SDOperand OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5876 SDOperand ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5877 SDOperand Vals[2] = { ResultVal, cpOutH.getValue(1) }; 5878 return DAG.getMergeValues(Vals, 2).Val; 5879} 5880 5881SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, SelectionDAG &DAG) { 5882 MVT T = Op->getValueType(0); 5883 assert (T == MVT::i32 && "Only know how to expand i32 Atomic Load Sub"); 5884 SDOperand negOp = DAG.getNode(ISD::SUB, T, 5885 DAG.getConstant(0, T), Op->getOperand(2)); 5886 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, Op->getOperand(0), 5887 Op->getOperand(1), negOp, 5888 cast<AtomicSDNode>(Op)->getSrcValue(), 5889 cast<AtomicSDNode>(Op)->getAlignment()).Val; 5890} 5891 5892/// LowerOperation - Provide custom lowering hooks for some operations. 5893/// 5894SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 5895 switch (Op.getOpcode()) { 5896 default: assert(0 && "Should not custom lower this!"); 5897 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 5898 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5899 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5900 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5901 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5902 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5903 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5904 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5905 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5906 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5907 case ISD::SHL_PARTS: 5908 case ISD::SRA_PARTS: 5909 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5910 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5911 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5912 case ISD::FABS: return LowerFABS(Op, DAG); 5913 case ISD::FNEG: return LowerFNEG(Op, DAG); 5914 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5915 case ISD::SETCC: return LowerSETCC(Op, DAG); 5916 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 5917 case ISD::SELECT: return LowerSELECT(Op, DAG); 5918 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5919 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5920 case ISD::CALL: return LowerCALL(Op, DAG); 5921 case ISD::RET: return LowerRET(Op, DAG); 5922 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 5923 case ISD::VASTART: return LowerVASTART(Op, DAG); 5924 case ISD::VAARG: return LowerVAARG(Op, DAG); 5925 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 5926 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 5927 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 5928 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 5929 case ISD::FRAME_TO_ARGS_OFFSET: 5930 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 5931 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 5932 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 5933 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 5934 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 5935 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 5936 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 5937 5938 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 5939 case ISD::READCYCLECOUNTER: 5940 return SDOperand(ExpandREADCYCLECOUNTER(Op.Val, DAG), 0); 5941 } 5942} 5943 5944/// ReplaceNodeResults - Replace a node with an illegal result type 5945/// with a new node built out of custom code. 5946SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) { 5947 switch (N->getOpcode()) { 5948 default: assert(0 && "Should not custom lower this!"); 5949 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 5950 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 5951 case ISD::ATOMIC_CMP_SWAP: return ExpandATOMIC_CMP_SWAP(N, DAG); 5952 case ISD::ATOMIC_LOAD_SUB: return ExpandATOMIC_LOAD_SUB(N,DAG); 5953 } 5954} 5955 5956const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 5957 switch (Opcode) { 5958 default: return NULL; 5959 case X86ISD::BSF: return "X86ISD::BSF"; 5960 case X86ISD::BSR: return "X86ISD::BSR"; 5961 case X86ISD::SHLD: return "X86ISD::SHLD"; 5962 case X86ISD::SHRD: return "X86ISD::SHRD"; 5963 case X86ISD::FAND: return "X86ISD::FAND"; 5964 case X86ISD::FOR: return "X86ISD::FOR"; 5965 case X86ISD::FXOR: return "X86ISD::FXOR"; 5966 case X86ISD::FSRL: return "X86ISD::FSRL"; 5967 case X86ISD::FILD: return "X86ISD::FILD"; 5968 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 5969 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 5970 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 5971 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 5972 case X86ISD::FLD: return "X86ISD::FLD"; 5973 case X86ISD::FST: return "X86ISD::FST"; 5974 case X86ISD::CALL: return "X86ISD::CALL"; 5975 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 5976 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 5977 case X86ISD::CMP: return "X86ISD::CMP"; 5978 case X86ISD::COMI: return "X86ISD::COMI"; 5979 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 5980 case X86ISD::SETCC: return "X86ISD::SETCC"; 5981 case X86ISD::CMOV: return "X86ISD::CMOV"; 5982 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 5983 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 5984 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 5985 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 5986 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 5987 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 5988 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 5989 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 5990 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 5991 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 5992 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 5993 case X86ISD::FMAX: return "X86ISD::FMAX"; 5994 case X86ISD::FMIN: return "X86ISD::FMIN"; 5995 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 5996 case X86ISD::FRCP: return "X86ISD::FRCP"; 5997 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 5998 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 5999 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 6000 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 6001 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 6002 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 6003 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 6004 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 6005 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 6006 case X86ISD::VSHL: return "X86ISD::VSHL"; 6007 case X86ISD::VSRL: return "X86ISD::VSRL"; 6008 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 6009 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 6010 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 6011 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 6012 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 6013 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 6014 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 6015 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 6016 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 6017 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 6018 } 6019} 6020 6021// isLegalAddressingMode - Return true if the addressing mode represented 6022// by AM is legal for this target, for a load/store of the specified type. 6023bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 6024 const Type *Ty) const { 6025 // X86 supports extremely general addressing modes. 6026 6027 // X86 allows a sign-extended 32-bit immediate field as a displacement. 6028 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 6029 return false; 6030 6031 if (AM.BaseGV) { 6032 // We can only fold this if we don't need an extra load. 6033 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 6034 return false; 6035 6036 // X86-64 only supports addr of globals in small code model. 6037 if (Subtarget->is64Bit()) { 6038 if (getTargetMachine().getCodeModel() != CodeModel::Small) 6039 return false; 6040 // If lower 4G is not available, then we must use rip-relative addressing. 6041 if (AM.BaseOffs || AM.Scale > 1) 6042 return false; 6043 } 6044 } 6045 6046 switch (AM.Scale) { 6047 case 0: 6048 case 1: 6049 case 2: 6050 case 4: 6051 case 8: 6052 // These scales always work. 6053 break; 6054 case 3: 6055 case 5: 6056 case 9: 6057 // These scales are formed with basereg+scalereg. Only accept if there is 6058 // no basereg yet. 6059 if (AM.HasBaseReg) 6060 return false; 6061 break; 6062 default: // Other stuff never works. 6063 return false; 6064 } 6065 6066 return true; 6067} 6068 6069 6070bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 6071 if (!Ty1->isInteger() || !Ty2->isInteger()) 6072 return false; 6073 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6074 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6075 if (NumBits1 <= NumBits2) 6076 return false; 6077 return Subtarget->is64Bit() || NumBits1 < 64; 6078} 6079 6080bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 6081 if (!VT1.isInteger() || !VT2.isInteger()) 6082 return false; 6083 unsigned NumBits1 = VT1.getSizeInBits(); 6084 unsigned NumBits2 = VT2.getSizeInBits(); 6085 if (NumBits1 <= NumBits2) 6086 return false; 6087 return Subtarget->is64Bit() || NumBits1 < 64; 6088} 6089 6090/// isShuffleMaskLegal - Targets can use this to indicate that they only 6091/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6092/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6093/// are assumed to be legal. 6094bool 6095X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT VT) const { 6096 // Only do shuffles on 128-bit vector types for now. 6097 if (VT.getSizeInBits() == 64) return false; 6098 return (Mask.Val->getNumOperands() <= 4 || 6099 isIdentityMask(Mask.Val) || 6100 isIdentityMask(Mask.Val, true) || 6101 isSplatMask(Mask.Val) || 6102 isPSHUFHW_PSHUFLWMask(Mask.Val) || 6103 X86::isUNPCKLMask(Mask.Val) || 6104 X86::isUNPCKHMask(Mask.Val) || 6105 X86::isUNPCKL_v_undef_Mask(Mask.Val) || 6106 X86::isUNPCKH_v_undef_Mask(Mask.Val)); 6107} 6108 6109bool 6110X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDOperand> &BVOps, 6111 MVT EVT, SelectionDAG &DAG) const { 6112 unsigned NumElts = BVOps.size(); 6113 // Only do shuffles on 128-bit vector types for now. 6114 if (EVT.getSizeInBits() * NumElts == 64) return false; 6115 if (NumElts == 2) return true; 6116 if (NumElts == 4) { 6117 return (isMOVLMask(&BVOps[0], 4) || 6118 isCommutedMOVL(&BVOps[0], 4, true) || 6119 isSHUFPMask(&BVOps[0], 4) || 6120 isCommutedSHUFP(&BVOps[0], 4)); 6121 } 6122 return false; 6123} 6124 6125//===----------------------------------------------------------------------===// 6126// X86 Scheduler Hooks 6127//===----------------------------------------------------------------------===// 6128 6129// private utility function 6130MachineBasicBlock * 6131X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 6132 MachineBasicBlock *MBB, 6133 unsigned regOpc, 6134 unsigned immOpc, 6135 bool invSrc) { 6136 // For the atomic bitwise operator, we generate 6137 // thisMBB: 6138 // newMBB: 6139 // ld t1 = [bitinstr.addr] 6140 // op t2 = t1, [bitinstr.val] 6141 // mov EAX = t1 6142 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6143 // bz newMBB 6144 // fallthrough -->nextMBB 6145 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6146 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6147 MachineFunction::iterator MBBIter = MBB; 6148 ++MBBIter; 6149 6150 /// First build the CFG 6151 MachineFunction *F = MBB->getParent(); 6152 MachineBasicBlock *thisMBB = MBB; 6153 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6154 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6155 F->insert(MBBIter, newMBB); 6156 F->insert(MBBIter, nextMBB); 6157 6158 // Move all successors to thisMBB to nextMBB 6159 nextMBB->transferSuccessors(thisMBB); 6160 6161 // Update thisMBB to fall through to newMBB 6162 thisMBB->addSuccessor(newMBB); 6163 6164 // newMBB jumps to itself and fall through to nextMBB 6165 newMBB->addSuccessor(nextMBB); 6166 newMBB->addSuccessor(newMBB); 6167 6168 // Insert instructions into newMBB based on incoming instruction 6169 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 6170 MachineOperand& destOper = bInstr->getOperand(0); 6171 MachineOperand* argOpers[6]; 6172 int numArgs = bInstr->getNumOperands() - 1; 6173 for (int i=0; i < numArgs; ++i) 6174 argOpers[i] = &bInstr->getOperand(i+1); 6175 6176 // x86 address has 4 operands: base, index, scale, and displacement 6177 int lastAddrIndx = 3; // [0,3] 6178 int valArgIndx = 4; 6179 6180 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6181 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6182 for (int i=0; i <= lastAddrIndx; ++i) 6183 (*MIB).addOperand(*argOpers[i]); 6184 6185 unsigned tt = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6186 if (invSrc) { 6187 MIB = BuildMI(newMBB, TII->get(X86::NOT32r), tt).addReg(t1); 6188 } 6189 else 6190 tt = t1; 6191 6192 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6193 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6194 && "invalid operand"); 6195 if (argOpers[valArgIndx]->isReg()) 6196 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6197 else 6198 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6199 MIB.addReg(tt); 6200 (*MIB).addOperand(*argOpers[valArgIndx]); 6201 6202 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6203 MIB.addReg(t1); 6204 6205 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6206 for (int i=0; i <= lastAddrIndx; ++i) 6207 (*MIB).addOperand(*argOpers[i]); 6208 MIB.addReg(t2); 6209 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6210 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 6211 6212 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6213 MIB.addReg(X86::EAX); 6214 6215 // insert branch 6216 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6217 6218 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 6219 return nextMBB; 6220} 6221 6222// private utility function 6223MachineBasicBlock * 6224X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6225 MachineBasicBlock *MBB, 6226 unsigned cmovOpc) { 6227 // For the atomic min/max operator, we generate 6228 // thisMBB: 6229 // newMBB: 6230 // ld t1 = [min/max.addr] 6231 // mov t2 = [min/max.val] 6232 // cmp t1, t2 6233 // cmov[cond] t2 = t1 6234 // mov EAX = t1 6235 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6236 // bz newMBB 6237 // fallthrough -->nextMBB 6238 // 6239 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6240 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6241 MachineFunction::iterator MBBIter = MBB; 6242 ++MBBIter; 6243 6244 /// First build the CFG 6245 MachineFunction *F = MBB->getParent(); 6246 MachineBasicBlock *thisMBB = MBB; 6247 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6248 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6249 F->insert(MBBIter, newMBB); 6250 F->insert(MBBIter, nextMBB); 6251 6252 // Move all successors to thisMBB to nextMBB 6253 nextMBB->transferSuccessors(thisMBB); 6254 6255 // Update thisMBB to fall through to newMBB 6256 thisMBB->addSuccessor(newMBB); 6257 6258 // newMBB jumps to newMBB and fall through to nextMBB 6259 newMBB->addSuccessor(nextMBB); 6260 newMBB->addSuccessor(newMBB); 6261 6262 // Insert instructions into newMBB based on incoming instruction 6263 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6264 MachineOperand& destOper = mInstr->getOperand(0); 6265 MachineOperand* argOpers[6]; 6266 int numArgs = mInstr->getNumOperands() - 1; 6267 for (int i=0; i < numArgs; ++i) 6268 argOpers[i] = &mInstr->getOperand(i+1); 6269 6270 // x86 address has 4 operands: base, index, scale, and displacement 6271 int lastAddrIndx = 3; // [0,3] 6272 int valArgIndx = 4; 6273 6274 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6275 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6276 for (int i=0; i <= lastAddrIndx; ++i) 6277 (*MIB).addOperand(*argOpers[i]); 6278 6279 // We only support register and immediate values 6280 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6281 && "invalid operand"); 6282 6283 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6284 if (argOpers[valArgIndx]->isReg()) 6285 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6286 else 6287 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6288 (*MIB).addOperand(*argOpers[valArgIndx]); 6289 6290 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6291 MIB.addReg(t1); 6292 6293 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6294 MIB.addReg(t1); 6295 MIB.addReg(t2); 6296 6297 // Generate movc 6298 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6299 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6300 MIB.addReg(t2); 6301 MIB.addReg(t1); 6302 6303 // Cmp and exchange if none has modified the memory location 6304 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6305 for (int i=0; i <= lastAddrIndx; ++i) 6306 (*MIB).addOperand(*argOpers[i]); 6307 MIB.addReg(t3); 6308 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6309 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 6310 6311 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6312 MIB.addReg(X86::EAX); 6313 6314 // insert branch 6315 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6316 6317 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 6318 return nextMBB; 6319} 6320 6321 6322MachineBasicBlock * 6323X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6324 MachineBasicBlock *BB) { 6325 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6326 switch (MI->getOpcode()) { 6327 default: assert(false && "Unexpected instr type to insert"); 6328 case X86::CMOV_FR32: 6329 case X86::CMOV_FR64: 6330 case X86::CMOV_V4F32: 6331 case X86::CMOV_V2F64: 6332 case X86::CMOV_V2I64: { 6333 // To "insert" a SELECT_CC instruction, we actually have to insert the 6334 // diamond control-flow pattern. The incoming instruction knows the 6335 // destination vreg to set, the condition code register to branch on, the 6336 // true/false values to select between, and a branch opcode to use. 6337 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6338 MachineFunction::iterator It = BB; 6339 ++It; 6340 6341 // thisMBB: 6342 // ... 6343 // TrueVal = ... 6344 // cmpTY ccX, r1, r2 6345 // bCC copy1MBB 6346 // fallthrough --> copy0MBB 6347 MachineBasicBlock *thisMBB = BB; 6348 MachineFunction *F = BB->getParent(); 6349 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6350 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6351 unsigned Opc = 6352 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6353 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6354 F->insert(It, copy0MBB); 6355 F->insert(It, sinkMBB); 6356 // Update machine-CFG edges by transferring all successors of the current 6357 // block to the new block which will contain the Phi node for the select. 6358 sinkMBB->transferSuccessors(BB); 6359 6360 // Add the true and fallthrough blocks as its successors. 6361 BB->addSuccessor(copy0MBB); 6362 BB->addSuccessor(sinkMBB); 6363 6364 // copy0MBB: 6365 // %FalseValue = ... 6366 // # fallthrough to sinkMBB 6367 BB = copy0MBB; 6368 6369 // Update machine-CFG edges 6370 BB->addSuccessor(sinkMBB); 6371 6372 // sinkMBB: 6373 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6374 // ... 6375 BB = sinkMBB; 6376 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6377 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6378 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6379 6380 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6381 return BB; 6382 } 6383 6384 case X86::FP32_TO_INT16_IN_MEM: 6385 case X86::FP32_TO_INT32_IN_MEM: 6386 case X86::FP32_TO_INT64_IN_MEM: 6387 case X86::FP64_TO_INT16_IN_MEM: 6388 case X86::FP64_TO_INT32_IN_MEM: 6389 case X86::FP64_TO_INT64_IN_MEM: 6390 case X86::FP80_TO_INT16_IN_MEM: 6391 case X86::FP80_TO_INT32_IN_MEM: 6392 case X86::FP80_TO_INT64_IN_MEM: { 6393 // Change the floating point control register to use "round towards zero" 6394 // mode when truncating to an integer value. 6395 MachineFunction *F = BB->getParent(); 6396 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6397 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6398 6399 // Load the old value of the high byte of the control word... 6400 unsigned OldCW = 6401 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6402 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6403 6404 // Set the high part to be round to zero... 6405 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6406 .addImm(0xC7F); 6407 6408 // Reload the modified control word now... 6409 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6410 6411 // Restore the memory image of control word to original value 6412 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6413 .addReg(OldCW); 6414 6415 // Get the X86 opcode to use. 6416 unsigned Opc; 6417 switch (MI->getOpcode()) { 6418 default: assert(0 && "illegal opcode!"); 6419 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6420 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6421 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6422 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6423 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6424 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6425 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6426 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6427 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6428 } 6429 6430 X86AddressMode AM; 6431 MachineOperand &Op = MI->getOperand(0); 6432 if (Op.isRegister()) { 6433 AM.BaseType = X86AddressMode::RegBase; 6434 AM.Base.Reg = Op.getReg(); 6435 } else { 6436 AM.BaseType = X86AddressMode::FrameIndexBase; 6437 AM.Base.FrameIndex = Op.getIndex(); 6438 } 6439 Op = MI->getOperand(1); 6440 if (Op.isImmediate()) 6441 AM.Scale = Op.getImm(); 6442 Op = MI->getOperand(2); 6443 if (Op.isImmediate()) 6444 AM.IndexReg = Op.getImm(); 6445 Op = MI->getOperand(3); 6446 if (Op.isGlobalAddress()) { 6447 AM.GV = Op.getGlobal(); 6448 } else { 6449 AM.Disp = Op.getImm(); 6450 } 6451 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6452 .addReg(MI->getOperand(4).getReg()); 6453 6454 // Reload the original control word now. 6455 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6456 6457 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6458 return BB; 6459 } 6460 case X86::ATOMAND32: 6461 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6462 X86::AND32ri); 6463 case X86::ATOMOR32: 6464 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6465 X86::OR32ri); 6466 case X86::ATOMXOR32: 6467 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6468 X86::XOR32ri); 6469 case X86::ATOMNAND32: 6470 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6471 X86::AND32ri, true); 6472 case X86::ATOMMIN32: 6473 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6474 case X86::ATOMMAX32: 6475 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6476 case X86::ATOMUMIN32: 6477 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6478 case X86::ATOMUMAX32: 6479 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6480 } 6481} 6482 6483//===----------------------------------------------------------------------===// 6484// X86 Optimization Hooks 6485//===----------------------------------------------------------------------===// 6486 6487void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op, 6488 const APInt &Mask, 6489 APInt &KnownZero, 6490 APInt &KnownOne, 6491 const SelectionDAG &DAG, 6492 unsigned Depth) const { 6493 unsigned Opc = Op.getOpcode(); 6494 assert((Opc >= ISD::BUILTIN_OP_END || 6495 Opc == ISD::INTRINSIC_WO_CHAIN || 6496 Opc == ISD::INTRINSIC_W_CHAIN || 6497 Opc == ISD::INTRINSIC_VOID) && 6498 "Should use MaskedValueIsZero if you don't know whether Op" 6499 " is a target node!"); 6500 6501 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6502 switch (Opc) { 6503 default: break; 6504 case X86ISD::SETCC: 6505 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6506 Mask.getBitWidth() - 1); 6507 break; 6508 } 6509} 6510 6511/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6512/// node is a GlobalAddress + offset. 6513bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6514 GlobalValue* &GA, int64_t &Offset) const{ 6515 if (N->getOpcode() == X86ISD::Wrapper) { 6516 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6517 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6518 return true; 6519 } 6520 } 6521 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6522} 6523 6524static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6525 const TargetLowering &TLI) { 6526 GlobalValue *GV; 6527 int64_t Offset = 0; 6528 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6529 return (GV->getAlignment() >= N && (Offset % N) == 0); 6530 // DAG combine handles the stack object case. 6531 return false; 6532} 6533 6534static bool EltsFromConsecutiveLoads(SDNode *N, SDOperand PermMask, 6535 unsigned NumElems, MVT EVT, 6536 SDNode *&Base, 6537 SelectionDAG &DAG, MachineFrameInfo *MFI, 6538 const TargetLowering &TLI) { 6539 Base = NULL; 6540 for (unsigned i = 0; i < NumElems; ++i) { 6541 SDOperand Idx = PermMask.getOperand(i); 6542 if (Idx.getOpcode() == ISD::UNDEF) { 6543 if (!Base) 6544 return false; 6545 continue; 6546 } 6547 6548 SDOperand Elt = DAG.getShuffleScalarElt(N, i); 6549 if (!Elt.Val || 6550 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.Val))) 6551 return false; 6552 if (!Base) { 6553 Base = Elt.Val; 6554 if (Base->getOpcode() == ISD::UNDEF) 6555 return false; 6556 continue; 6557 } 6558 if (Elt.getOpcode() == ISD::UNDEF) 6559 continue; 6560 6561 if (!TLI.isConsecutiveLoad(Elt.Val, Base, 6562 EVT.getSizeInBits()/8, i, MFI)) 6563 return false; 6564 } 6565 return true; 6566} 6567 6568/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6569/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6570/// if the load addresses are consecutive, non-overlapping, and in the right 6571/// order. 6572static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6573 const TargetLowering &TLI) { 6574 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6575 MVT VT = N->getValueType(0); 6576 MVT EVT = VT.getVectorElementType(); 6577 SDOperand PermMask = N->getOperand(2); 6578 unsigned NumElems = PermMask.getNumOperands(); 6579 SDNode *Base = NULL; 6580 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6581 DAG, MFI, TLI)) 6582 return SDOperand(); 6583 6584 LoadSDNode *LD = cast<LoadSDNode>(Base); 6585 if (isBaseAlignmentOfN(16, Base->getOperand(1).Val, TLI)) 6586 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6587 LD->getSrcValueOffset(), LD->isVolatile()); 6588 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6589 LD->getSrcValueOffset(), LD->isVolatile(), 6590 LD->getAlignment()); 6591} 6592 6593/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6594static SDOperand PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6595 const X86Subtarget *Subtarget, 6596 const TargetLowering &TLI) { 6597 unsigned NumOps = N->getNumOperands(); 6598 6599 // Ignore single operand BUILD_VECTOR. 6600 if (NumOps == 1) 6601 return SDOperand(); 6602 6603 MVT VT = N->getValueType(0); 6604 MVT EVT = VT.getVectorElementType(); 6605 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6606 // We are looking for load i64 and zero extend. We want to transform 6607 // it before legalizer has a chance to expand it. Also look for i64 6608 // BUILD_PAIR bit casted to f64. 6609 return SDOperand(); 6610 // This must be an insertion into a zero vector. 6611 SDOperand HighElt = N->getOperand(1); 6612 if (!isZeroNode(HighElt)) 6613 return SDOperand(); 6614 6615 // Value must be a load. 6616 SDNode *Base = N->getOperand(0).Val; 6617 if (!isa<LoadSDNode>(Base)) { 6618 if (Base->getOpcode() != ISD::BIT_CONVERT) 6619 return SDOperand(); 6620 Base = Base->getOperand(0).Val; 6621 if (!isa<LoadSDNode>(Base)) 6622 return SDOperand(); 6623 } 6624 6625 // Transform it into VZEXT_LOAD addr. 6626 LoadSDNode *LD = cast<LoadSDNode>(Base); 6627 6628 // Load must not be an extload. 6629 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6630 return SDOperand(); 6631 6632 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6633} 6634 6635/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6636static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6637 const X86Subtarget *Subtarget) { 6638 SDOperand Cond = N->getOperand(0); 6639 6640 // If we have SSE[12] support, try to form min/max nodes. 6641 if (Subtarget->hasSSE2() && 6642 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6643 if (Cond.getOpcode() == ISD::SETCC) { 6644 // Get the LHS/RHS of the select. 6645 SDOperand LHS = N->getOperand(1); 6646 SDOperand RHS = N->getOperand(2); 6647 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6648 6649 unsigned Opcode = 0; 6650 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6651 switch (CC) { 6652 default: break; 6653 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6654 case ISD::SETULE: 6655 case ISD::SETLE: 6656 if (!UnsafeFPMath) break; 6657 // FALL THROUGH. 6658 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6659 case ISD::SETLT: 6660 Opcode = X86ISD::FMIN; 6661 break; 6662 6663 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6664 case ISD::SETUGT: 6665 case ISD::SETGT: 6666 if (!UnsafeFPMath) break; 6667 // FALL THROUGH. 6668 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6669 case ISD::SETGE: 6670 Opcode = X86ISD::FMAX; 6671 break; 6672 } 6673 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6674 switch (CC) { 6675 default: break; 6676 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6677 case ISD::SETUGT: 6678 case ISD::SETGT: 6679 if (!UnsafeFPMath) break; 6680 // FALL THROUGH. 6681 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6682 case ISD::SETGE: 6683 Opcode = X86ISD::FMIN; 6684 break; 6685 6686 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6687 case ISD::SETULE: 6688 case ISD::SETLE: 6689 if (!UnsafeFPMath) break; 6690 // FALL THROUGH. 6691 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6692 case ISD::SETLT: 6693 Opcode = X86ISD::FMAX; 6694 break; 6695 } 6696 } 6697 6698 if (Opcode) 6699 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6700 } 6701 6702 } 6703 6704 return SDOperand(); 6705} 6706 6707/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6708static SDOperand PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6709 const X86Subtarget *Subtarget) { 6710 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6711 // the FP state in cases where an emms may be missing. 6712 // A preferable solution to the general problem is to figure out the right 6713 // places to insert EMMS. This qualifies as a quick hack. 6714 StoreSDNode *St = cast<StoreSDNode>(N); 6715 if (St->getValue().getValueType().isVector() && 6716 St->getValue().getValueType().getSizeInBits() == 64 && 6717 isa<LoadSDNode>(St->getValue()) && 6718 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6719 St->getChain().hasOneUse() && !St->isVolatile()) { 6720 SDNode* LdVal = St->getValue().Val; 6721 LoadSDNode *Ld = 0; 6722 int TokenFactorIndex = -1; 6723 SmallVector<SDOperand, 8> Ops; 6724 SDNode* ChainVal = St->getChain().Val; 6725 // Must be a store of a load. We currently handle two cases: the load 6726 // is a direct child, and it's under an intervening TokenFactor. It is 6727 // possible to dig deeper under nested TokenFactors. 6728 if (ChainVal == LdVal) 6729 Ld = cast<LoadSDNode>(St->getChain()); 6730 else if (St->getValue().hasOneUse() && 6731 ChainVal->getOpcode() == ISD::TokenFactor) { 6732 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6733 if (ChainVal->getOperand(i).Val == LdVal) { 6734 TokenFactorIndex = i; 6735 Ld = cast<LoadSDNode>(St->getValue()); 6736 } else 6737 Ops.push_back(ChainVal->getOperand(i)); 6738 } 6739 } 6740 if (Ld) { 6741 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6742 if (Subtarget->is64Bit()) { 6743 SDOperand NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6744 Ld->getBasePtr(), Ld->getSrcValue(), 6745 Ld->getSrcValueOffset(), Ld->isVolatile(), 6746 Ld->getAlignment()); 6747 SDOperand NewChain = NewLd.getValue(1); 6748 if (TokenFactorIndex != -1) { 6749 Ops.push_back(NewChain); 6750 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6751 Ops.size()); 6752 } 6753 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6754 St->getSrcValue(), St->getSrcValueOffset(), 6755 St->isVolatile(), St->getAlignment()); 6756 } 6757 6758 // Otherwise, lower to two 32-bit copies. 6759 SDOperand LoAddr = Ld->getBasePtr(); 6760 SDOperand HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6761 DAG.getConstant(4, MVT::i32)); 6762 6763 SDOperand LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6764 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6765 Ld->isVolatile(), Ld->getAlignment()); 6766 SDOperand HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6767 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6768 Ld->isVolatile(), 6769 MinAlign(Ld->getAlignment(), 4)); 6770 6771 SDOperand NewChain = LoLd.getValue(1); 6772 if (TokenFactorIndex != -1) { 6773 Ops.push_back(LoLd); 6774 Ops.push_back(HiLd); 6775 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6776 Ops.size()); 6777 } 6778 6779 LoAddr = St->getBasePtr(); 6780 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6781 DAG.getConstant(4, MVT::i32)); 6782 6783 SDOperand LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6784 St->getSrcValue(), St->getSrcValueOffset(), 6785 St->isVolatile(), St->getAlignment()); 6786 SDOperand HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6787 St->getSrcValue(), St->getSrcValueOffset()+4, 6788 St->isVolatile(), 6789 MinAlign(St->getAlignment(), 4)); 6790 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6791 } 6792 } 6793 return SDOperand(); 6794} 6795 6796/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6797/// X86ISD::FXOR nodes. 6798static SDOperand PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6799 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6800 // F[X]OR(0.0, x) -> x 6801 // F[X]OR(x, 0.0) -> x 6802 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6803 if (C->getValueAPF().isPosZero()) 6804 return N->getOperand(1); 6805 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6806 if (C->getValueAPF().isPosZero()) 6807 return N->getOperand(0); 6808 return SDOperand(); 6809} 6810 6811/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 6812static SDOperand PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 6813 // FAND(0.0, x) -> 0.0 6814 // FAND(x, 0.0) -> 0.0 6815 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6816 if (C->getValueAPF().isPosZero()) 6817 return N->getOperand(0); 6818 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6819 if (C->getValueAPF().isPosZero()) 6820 return N->getOperand(1); 6821 return SDOperand(); 6822} 6823 6824 6825SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N, 6826 DAGCombinerInfo &DCI) const { 6827 SelectionDAG &DAG = DCI.DAG; 6828 switch (N->getOpcode()) { 6829 default: break; 6830 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 6831 case ISD::BUILD_VECTOR: 6832 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 6833 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 6834 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 6835 case X86ISD::FXOR: 6836 case X86ISD::FOR: return PerformFORCombine(N, DAG); 6837 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 6838 } 6839 6840 return SDOperand(); 6841} 6842 6843//===----------------------------------------------------------------------===// 6844// X86 Inline Assembly Support 6845//===----------------------------------------------------------------------===// 6846 6847/// getConstraintType - Given a constraint letter, return the type of 6848/// constraint it is for this target. 6849X86TargetLowering::ConstraintType 6850X86TargetLowering::getConstraintType(const std::string &Constraint) const { 6851 if (Constraint.size() == 1) { 6852 switch (Constraint[0]) { 6853 case 'A': 6854 case 'f': 6855 case 'r': 6856 case 'R': 6857 case 'l': 6858 case 'q': 6859 case 'Q': 6860 case 'x': 6861 case 'y': 6862 case 'Y': 6863 return C_RegisterClass; 6864 default: 6865 break; 6866 } 6867 } 6868 return TargetLowering::getConstraintType(Constraint); 6869} 6870 6871/// LowerXConstraint - try to replace an X constraint, which matches anything, 6872/// with another that has more specific requirements based on the type of the 6873/// corresponding operand. 6874const char *X86TargetLowering:: 6875LowerXConstraint(MVT ConstraintVT) const { 6876 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 6877 // 'f' like normal targets. 6878 if (ConstraintVT.isFloatingPoint()) { 6879 if (Subtarget->hasSSE2()) 6880 return "Y"; 6881 if (Subtarget->hasSSE1()) 6882 return "x"; 6883 } 6884 6885 return TargetLowering::LowerXConstraint(ConstraintVT); 6886} 6887 6888/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 6889/// vector. If it is invalid, don't add anything to Ops. 6890void X86TargetLowering::LowerAsmOperandForConstraint(SDOperand Op, 6891 char Constraint, 6892 std::vector<SDOperand>&Ops, 6893 SelectionDAG &DAG) const { 6894 SDOperand Result(0, 0); 6895 6896 switch (Constraint) { 6897 default: break; 6898 case 'I': 6899 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6900 if (C->getValue() <= 31) { 6901 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6902 break; 6903 } 6904 } 6905 return; 6906 case 'N': 6907 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6908 if (C->getValue() <= 255) { 6909 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6910 break; 6911 } 6912 } 6913 return; 6914 case 'i': { 6915 // Literal immediates are always ok. 6916 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 6917 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 6918 break; 6919 } 6920 6921 // If we are in non-pic codegen mode, we allow the address of a global (with 6922 // an optional displacement) to be used with 'i'. 6923 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 6924 int64_t Offset = 0; 6925 6926 // Match either (GA) or (GA+C) 6927 if (GA) { 6928 Offset = GA->getOffset(); 6929 } else if (Op.getOpcode() == ISD::ADD) { 6930 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6931 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6932 if (C && GA) { 6933 Offset = GA->getOffset()+C->getValue(); 6934 } else { 6935 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6936 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6937 if (C && GA) 6938 Offset = GA->getOffset()+C->getValue(); 6939 else 6940 C = 0, GA = 0; 6941 } 6942 } 6943 6944 if (GA) { 6945 // If addressing this global requires a load (e.g. in PIC mode), we can't 6946 // match. 6947 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 6948 false)) 6949 return; 6950 6951 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 6952 Offset); 6953 Result = Op; 6954 break; 6955 } 6956 6957 // Otherwise, not valid for this mode. 6958 return; 6959 } 6960 } 6961 6962 if (Result.Val) { 6963 Ops.push_back(Result); 6964 return; 6965 } 6966 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 6967} 6968 6969std::vector<unsigned> X86TargetLowering:: 6970getRegClassForInlineAsmConstraint(const std::string &Constraint, 6971 MVT VT) const { 6972 if (Constraint.size() == 1) { 6973 // FIXME: not handling fp-stack yet! 6974 switch (Constraint[0]) { // GCC X86 Constraint Letters 6975 default: break; // Unknown constraint letter 6976 case 'A': // EAX/EDX 6977 if (VT == MVT::i32 || VT == MVT::i64) 6978 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 6979 break; 6980 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 6981 case 'Q': // Q_REGS 6982 if (VT == MVT::i32) 6983 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 6984 else if (VT == MVT::i16) 6985 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 6986 else if (VT == MVT::i8) 6987 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 6988 else if (VT == MVT::i64) 6989 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 6990 break; 6991 } 6992 } 6993 6994 return std::vector<unsigned>(); 6995} 6996 6997std::pair<unsigned, const TargetRegisterClass*> 6998X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 6999 MVT VT) const { 7000 // First, see if this is a constraint that directly corresponds to an LLVM 7001 // register class. 7002 if (Constraint.size() == 1) { 7003 // GCC Constraint Letters 7004 switch (Constraint[0]) { 7005 default: break; 7006 case 'r': // GENERAL_REGS 7007 case 'R': // LEGACY_REGS 7008 case 'l': // INDEX_REGS 7009 if (VT == MVT::i64 && Subtarget->is64Bit()) 7010 return std::make_pair(0U, X86::GR64RegisterClass); 7011 if (VT == MVT::i32) 7012 return std::make_pair(0U, X86::GR32RegisterClass); 7013 else if (VT == MVT::i16) 7014 return std::make_pair(0U, X86::GR16RegisterClass); 7015 else if (VT == MVT::i8) 7016 return std::make_pair(0U, X86::GR8RegisterClass); 7017 break; 7018 case 'f': // FP Stack registers. 7019 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 7020 // value to the correct fpstack register class. 7021 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 7022 return std::make_pair(0U, X86::RFP32RegisterClass); 7023 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 7024 return std::make_pair(0U, X86::RFP64RegisterClass); 7025 return std::make_pair(0U, X86::RFP80RegisterClass); 7026 case 'y': // MMX_REGS if MMX allowed. 7027 if (!Subtarget->hasMMX()) break; 7028 return std::make_pair(0U, X86::VR64RegisterClass); 7029 break; 7030 case 'Y': // SSE_REGS if SSE2 allowed 7031 if (!Subtarget->hasSSE2()) break; 7032 // FALL THROUGH. 7033 case 'x': // SSE_REGS if SSE1 allowed 7034 if (!Subtarget->hasSSE1()) break; 7035 7036 switch (VT.getSimpleVT()) { 7037 default: break; 7038 // Scalar SSE types. 7039 case MVT::f32: 7040 case MVT::i32: 7041 return std::make_pair(0U, X86::FR32RegisterClass); 7042 case MVT::f64: 7043 case MVT::i64: 7044 return std::make_pair(0U, X86::FR64RegisterClass); 7045 // Vector types. 7046 case MVT::v16i8: 7047 case MVT::v8i16: 7048 case MVT::v4i32: 7049 case MVT::v2i64: 7050 case MVT::v4f32: 7051 case MVT::v2f64: 7052 return std::make_pair(0U, X86::VR128RegisterClass); 7053 } 7054 break; 7055 } 7056 } 7057 7058 // Use the default implementation in TargetLowering to convert the register 7059 // constraint into a member of a register class. 7060 std::pair<unsigned, const TargetRegisterClass*> Res; 7061 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 7062 7063 // Not found as a standard register? 7064 if (Res.second == 0) { 7065 // GCC calls "st(0)" just plain "st". 7066 if (StringsEqualNoCase("{st}", Constraint)) { 7067 Res.first = X86::ST0; 7068 Res.second = X86::RFP80RegisterClass; 7069 } 7070 7071 return Res; 7072 } 7073 7074 // Otherwise, check to see if this is a register class of the wrong value 7075 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 7076 // turn into {ax},{dx}. 7077 if (Res.second->hasType(VT)) 7078 return Res; // Correct type already, nothing to do. 7079 7080 // All of the single-register GCC register classes map their values onto 7081 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 7082 // really want an 8-bit or 32-bit register, map to the appropriate register 7083 // class and return the appropriate register. 7084 if (Res.second != X86::GR16RegisterClass) 7085 return Res; 7086 7087 if (VT == MVT::i8) { 7088 unsigned DestReg = 0; 7089 switch (Res.first) { 7090 default: break; 7091 case X86::AX: DestReg = X86::AL; break; 7092 case X86::DX: DestReg = X86::DL; break; 7093 case X86::CX: DestReg = X86::CL; break; 7094 case X86::BX: DestReg = X86::BL; break; 7095 } 7096 if (DestReg) { 7097 Res.first = DestReg; 7098 Res.second = Res.second = X86::GR8RegisterClass; 7099 } 7100 } else if (VT == MVT::i32) { 7101 unsigned DestReg = 0; 7102 switch (Res.first) { 7103 default: break; 7104 case X86::AX: DestReg = X86::EAX; break; 7105 case X86::DX: DestReg = X86::EDX; break; 7106 case X86::CX: DestReg = X86::ECX; break; 7107 case X86::BX: DestReg = X86::EBX; break; 7108 case X86::SI: DestReg = X86::ESI; break; 7109 case X86::DI: DestReg = X86::EDI; break; 7110 case X86::BP: DestReg = X86::EBP; break; 7111 case X86::SP: DestReg = X86::ESP; break; 7112 } 7113 if (DestReg) { 7114 Res.first = DestReg; 7115 Res.second = Res.second = X86::GR32RegisterClass; 7116 } 7117 } else if (VT == MVT::i64) { 7118 unsigned DestReg = 0; 7119 switch (Res.first) { 7120 default: break; 7121 case X86::AX: DestReg = X86::RAX; break; 7122 case X86::DX: DestReg = X86::RDX; break; 7123 case X86::CX: DestReg = X86::RCX; break; 7124 case X86::BX: DestReg = X86::RBX; break; 7125 case X86::SI: DestReg = X86::RSI; break; 7126 case X86::DI: DestReg = X86::RDI; break; 7127 case X86::BP: DestReg = X86::RBP; break; 7128 case X86::SP: DestReg = X86::RSP; break; 7129 } 7130 if (DestReg) { 7131 Res.first = DestReg; 7132 Res.second = Res.second = X86::GR64RegisterClass; 7133 } 7134 } 7135 7136 return Res; 7137} 7138