X86ISelLowering.cpp revision 1fdc40f0622375f7f9a4ccfbbf5deda68131f6e0
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86TargetMachine.h" 20#include "llvm/CallingConv.h" 21#include "llvm/Constants.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/GlobalVariable.h" 24#include "llvm/Function.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/ADT/BitVector.h" 27#include "llvm/ADT/VectorExtras.h" 28#include "llvm/CodeGen/CallingConvLower.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/PseudoSourceValue.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/Debug.h" 38#include "llvm/Target/TargetOptions.h" 39#include "llvm/ADT/SmallSet.h" 40#include "llvm/ADT/StringExtras.h" 41using namespace llvm; 42 43// Forward declarations. 44static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45 46X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47 : TargetLowering(TM) { 48 Subtarget = &TM.getSubtarget<X86Subtarget>(); 49 X86ScalarSSEf64 = Subtarget->hasSSE2(); 50 X86ScalarSSEf32 = Subtarget->hasSSE1(); 51 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52 53 bool Fast = false; 54 55 RegInfo = TM.getRegisterInfo(); 56 TD = getTargetData(); 57 58 // Set up the TargetLowering object. 59 60 // X86 is weird, it always uses i8 for shift amounts and setcc results. 61 setShiftAmountType(MVT::i8); 62 setSetCCResultContents(ZeroOrOneSetCCResult); 63 setSchedulingPreference(SchedulingForRegPressure); 64 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 65 setStackPointerRegisterToSaveRestore(X86StackPtr); 66 67 if (Subtarget->isTargetDarwin()) { 68 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 69 setUseUnderscoreSetJmp(false); 70 setUseUnderscoreLongJmp(false); 71 } else if (Subtarget->isTargetMingw()) { 72 // MS runtime is weird: it exports _setjmp, but longjmp! 73 setUseUnderscoreSetJmp(true); 74 setUseUnderscoreLongJmp(false); 75 } else { 76 setUseUnderscoreSetJmp(true); 77 setUseUnderscoreLongJmp(true); 78 } 79 80 // Set up the register classes. 81 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 82 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 83 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 84 if (Subtarget->is64Bit()) 85 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 86 87 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 88 89 // We don't accept any truncstore of integer registers. 90 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 91 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 92 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 93 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 94 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 95 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 96 97 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 98 // operation. 99 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 100 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 101 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 102 103 if (Subtarget->is64Bit()) { 104 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 105 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 106 } else { 107 if (X86ScalarSSEf64) 108 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 109 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 110 else 111 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 112 } 113 114 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 115 // this operation. 116 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 117 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 118 // SSE has no i16 to fp conversion, only i32 119 if (X86ScalarSSEf32) { 120 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 121 // f32 and f64 cases are Legal, f80 case is not 122 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 123 } else { 124 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 125 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 126 } 127 128 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 129 // are Legal, f80 is custom lowered. 130 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 131 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 132 133 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 134 // this operation. 135 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 136 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 137 138 if (X86ScalarSSEf32) { 139 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 140 // f32 and f64 cases are Legal, f80 case is not 141 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 142 } else { 143 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 144 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 145 } 146 147 // Handle FP_TO_UINT by promoting the destination to a larger signed 148 // conversion. 149 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 151 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 152 153 if (Subtarget->is64Bit()) { 154 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 155 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 156 } else { 157 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 158 // Expand FP_TO_UINT into a select. 159 // FIXME: We would like to use a Custom expander here eventually to do 160 // the optimal thing for SSE vs. the default expansion in the legalizer. 161 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 162 else 163 // With SSE3 we can use fisttpll to convert to a signed i64. 164 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 165 } 166 167 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 168 if (!X86ScalarSSEf64) { 169 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 170 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 171 } 172 173 // Scalar integer divide and remainder are lowered to use operations that 174 // produce two results, to match the available instructions. This exposes 175 // the two-result form to trivial CSE, which is able to combine x/y and x%y 176 // into a single instruction. 177 // 178 // Scalar integer multiply-high is also lowered to use two-result 179 // operations, to match the available instructions. However, plain multiply 180 // (low) operations are left as Legal, as there are single-result 181 // instructions for this in x86. Using the two-result multiply instructions 182 // when both high and low results are needed must be arranged by dagcombine. 183 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 184 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 185 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 187 setOperationAction(ISD::SREM , MVT::i8 , Expand); 188 setOperationAction(ISD::UREM , MVT::i8 , Expand); 189 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 190 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 191 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 193 setOperationAction(ISD::SREM , MVT::i16 , Expand); 194 setOperationAction(ISD::UREM , MVT::i16 , Expand); 195 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 196 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 197 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 199 setOperationAction(ISD::SREM , MVT::i32 , Expand); 200 setOperationAction(ISD::UREM , MVT::i32 , Expand); 201 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 202 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 203 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 205 setOperationAction(ISD::SREM , MVT::i64 , Expand); 206 setOperationAction(ISD::UREM , MVT::i64 , Expand); 207 208 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 209 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 210 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 211 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 212 if (Subtarget->is64Bit()) 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 216 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 217 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f32 , Expand); 219 setOperationAction(ISD::FREM , MVT::f64 , Expand); 220 setOperationAction(ISD::FREM , MVT::f80 , Expand); 221 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 222 223 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 224 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 226 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 227 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 229 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 230 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 231 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 232 if (Subtarget->is64Bit()) { 233 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 234 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 235 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 236 } 237 238 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 239 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 240 241 // These should be promoted to a larger select which is supported. 242 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 243 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 244 // X86 wants to expand cmov itself. 245 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 246 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 249 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 252 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 255 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 256 if (Subtarget->is64Bit()) { 257 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 258 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 259 } 260 // X86 ret instruction may pop stack. 261 setOperationAction(ISD::RET , MVT::Other, Custom); 262 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 263 264 // Darwin ABI issue. 265 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 266 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 268 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 269 if (Subtarget->is64Bit()) 270 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 271 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 272 if (Subtarget->is64Bit()) { 273 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 274 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 275 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 276 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 277 } 278 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 279 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 281 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 282 if (Subtarget->is64Bit()) { 283 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 285 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 286 } 287 288 if (Subtarget->hasSSE1()) 289 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 290 291 if (!Subtarget->hasSSE2()) 292 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 293 294 // Expand certain atomics 295 setOperationAction(ISD::ATOMIC_CMP_SWAP_8 , MVT::i8, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP_16, MVT::i16, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP_32, MVT::i32, Custom); 298 setOperationAction(ISD::ATOMIC_CMP_SWAP_64, MVT::i64, Custom); 299 300 setOperationAction(ISD::ATOMIC_LOAD_SUB_8, MVT::i8, Expand); 301 setOperationAction(ISD::ATOMIC_LOAD_SUB_16, MVT::i16, Expand); 302 setOperationAction(ISD::ATOMIC_LOAD_SUB_32, MVT::i32, Expand); 303 setOperationAction(ISD::ATOMIC_LOAD_SUB_64, MVT::i64, Expand); 304 305 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 306 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 307 // FIXME - use subtarget debug flags 308 if (!Subtarget->isTargetDarwin() && 309 !Subtarget->isTargetELF() && 310 !Subtarget->isTargetCygMing()) { 311 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 312 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 313 } 314 315 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 316 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 317 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 318 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 319 if (Subtarget->is64Bit()) { 320 setExceptionPointerRegister(X86::RAX); 321 setExceptionSelectorRegister(X86::RDX); 322 } else { 323 setExceptionPointerRegister(X86::EAX); 324 setExceptionSelectorRegister(X86::EDX); 325 } 326 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 327 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 328 329 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 330 331 setOperationAction(ISD::TRAP, MVT::Other, Legal); 332 333 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 334 setOperationAction(ISD::VASTART , MVT::Other, Custom); 335 setOperationAction(ISD::VAEND , MVT::Other, Expand); 336 if (Subtarget->is64Bit()) { 337 setOperationAction(ISD::VAARG , MVT::Other, Custom); 338 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 339 } else { 340 setOperationAction(ISD::VAARG , MVT::Other, Expand); 341 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 342 } 343 344 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 345 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 346 if (Subtarget->is64Bit()) 347 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 348 if (Subtarget->isTargetCygMing()) 349 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 350 else 351 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 352 353 if (X86ScalarSSEf64) { 354 // f32 and f64 use SSE. 355 // Set up the FP register classes. 356 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 357 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 358 359 // Use ANDPD to simulate FABS. 360 setOperationAction(ISD::FABS , MVT::f64, Custom); 361 setOperationAction(ISD::FABS , MVT::f32, Custom); 362 363 // Use XORP to simulate FNEG. 364 setOperationAction(ISD::FNEG , MVT::f64, Custom); 365 setOperationAction(ISD::FNEG , MVT::f32, Custom); 366 367 // Use ANDPD and ORPD to simulate FCOPYSIGN. 368 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 369 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 370 371 // We don't support sin/cos/fmod 372 setOperationAction(ISD::FSIN , MVT::f64, Expand); 373 setOperationAction(ISD::FCOS , MVT::f64, Expand); 374 setOperationAction(ISD::FSIN , MVT::f32, Expand); 375 setOperationAction(ISD::FCOS , MVT::f32, Expand); 376 377 // Expand FP immediates into loads from the stack, except for the special 378 // cases we handle. 379 addLegalFPImmediate(APFloat(+0.0)); // xorpd 380 addLegalFPImmediate(APFloat(+0.0f)); // xorps 381 382 // Floating truncations from f80 and extensions to f80 go through memory. 383 // If optimizing, we lie about this though and handle it in 384 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 385 if (Fast) { 386 setConvertAction(MVT::f32, MVT::f80, Expand); 387 setConvertAction(MVT::f64, MVT::f80, Expand); 388 setConvertAction(MVT::f80, MVT::f32, Expand); 389 setConvertAction(MVT::f80, MVT::f64, Expand); 390 } 391 } else if (X86ScalarSSEf32) { 392 // Use SSE for f32, x87 for f64. 393 // Set up the FP register classes. 394 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 395 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 396 397 // Use ANDPS to simulate FABS. 398 setOperationAction(ISD::FABS , MVT::f32, Custom); 399 400 // Use XORP to simulate FNEG. 401 setOperationAction(ISD::FNEG , MVT::f32, Custom); 402 403 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 404 405 // Use ANDPS and ORPS to simulate FCOPYSIGN. 406 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 407 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 408 409 // We don't support sin/cos/fmod 410 setOperationAction(ISD::FSIN , MVT::f32, Expand); 411 setOperationAction(ISD::FCOS , MVT::f32, Expand); 412 413 // Special cases we handle for FP constants. 414 addLegalFPImmediate(APFloat(+0.0f)); // xorps 415 addLegalFPImmediate(APFloat(+0.0)); // FLD0 416 addLegalFPImmediate(APFloat(+1.0)); // FLD1 417 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 418 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 419 420 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 421 // this though and handle it in InstructionSelectPreprocess so that 422 // dagcombine2 can hack on these. 423 if (Fast) { 424 setConvertAction(MVT::f32, MVT::f64, Expand); 425 setConvertAction(MVT::f32, MVT::f80, Expand); 426 setConvertAction(MVT::f80, MVT::f32, Expand); 427 setConvertAction(MVT::f64, MVT::f32, Expand); 428 // And x87->x87 truncations also. 429 setConvertAction(MVT::f80, MVT::f64, Expand); 430 } 431 432 if (!UnsafeFPMath) { 433 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 434 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 435 } 436 } else { 437 // f32 and f64 in x87. 438 // Set up the FP register classes. 439 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 440 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 441 442 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 443 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 444 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 445 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 446 447 // Floating truncations go through memory. If optimizing, we lie about 448 // this though and handle it in InstructionSelectPreprocess so that 449 // dagcombine2 can hack on these. 450 if (Fast) { 451 setConvertAction(MVT::f80, MVT::f32, Expand); 452 setConvertAction(MVT::f64, MVT::f32, Expand); 453 setConvertAction(MVT::f80, MVT::f64, Expand); 454 } 455 456 if (!UnsafeFPMath) { 457 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 458 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 459 } 460 addLegalFPImmediate(APFloat(+0.0)); // FLD0 461 addLegalFPImmediate(APFloat(+1.0)); // FLD1 462 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 463 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 464 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 465 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 466 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 467 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 468 } 469 470 // Long double always uses X87. 471 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 472 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 473 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 474 { 475 APFloat TmpFlt(+0.0); 476 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 477 addLegalFPImmediate(TmpFlt); // FLD0 478 TmpFlt.changeSign(); 479 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 480 APFloat TmpFlt2(+1.0); 481 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 482 addLegalFPImmediate(TmpFlt2); // FLD1 483 TmpFlt2.changeSign(); 484 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 485 } 486 487 if (!UnsafeFPMath) { 488 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 489 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 490 } 491 492 // Always use a library call for pow. 493 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 494 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 495 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 496 497 setOperationAction(ISD::FLOG, MVT::f32, Expand); 498 setOperationAction(ISD::FLOG, MVT::f64, Expand); 499 setOperationAction(ISD::FLOG, MVT::f80, Expand); 500 setOperationAction(ISD::FLOG2, MVT::f32, Expand); 501 setOperationAction(ISD::FLOG2, MVT::f64, Expand); 502 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 503 setOperationAction(ISD::FLOG10, MVT::f32, Expand); 504 setOperationAction(ISD::FLOG10, MVT::f64, Expand); 505 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 506 setOperationAction(ISD::FEXP, MVT::f32, Expand); 507 setOperationAction(ISD::FEXP, MVT::f64, Expand); 508 setOperationAction(ISD::FEXP, MVT::f80, Expand); 509 setOperationAction(ISD::FEXP2, MVT::f32, Expand); 510 setOperationAction(ISD::FEXP2, MVT::f64, Expand); 511 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 512 513 // First set operation action for all vector types to expand. Then we 514 // will selectively turn on ones that can be effectively codegen'd. 515 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 516 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 517 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); 532 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); 533 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 534 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 535 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 536 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 537 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 538 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 539 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 540 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 541 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 542 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 543 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 544 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 545 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 546 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 547 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 548 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 549 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 550 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 551 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 552 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 553 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 554 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 555 setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand); 556 setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand); 557 setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand); 558 setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand); 559 setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand); 560 } 561 562 if (Subtarget->hasMMX()) { 563 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 564 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 565 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 566 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 567 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 568 569 // FIXME: add MMX packed arithmetics 570 571 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 572 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 573 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 574 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 575 576 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 577 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 578 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 579 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 580 581 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 582 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 583 584 setOperationAction(ISD::AND, MVT::v8i8, Promote); 585 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 586 setOperationAction(ISD::AND, MVT::v4i16, Promote); 587 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 588 setOperationAction(ISD::AND, MVT::v2i32, Promote); 589 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 590 setOperationAction(ISD::AND, MVT::v1i64, Legal); 591 592 setOperationAction(ISD::OR, MVT::v8i8, Promote); 593 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 594 setOperationAction(ISD::OR, MVT::v4i16, Promote); 595 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 596 setOperationAction(ISD::OR, MVT::v2i32, Promote); 597 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 598 setOperationAction(ISD::OR, MVT::v1i64, Legal); 599 600 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 601 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 602 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 603 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 604 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 605 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 606 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 607 608 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 609 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 610 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 611 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 612 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 613 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 614 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 615 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 616 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 617 618 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 619 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 620 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 621 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 622 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 623 624 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 625 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 626 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 627 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 628 629 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); 630 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 631 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 632 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 633 634 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 635 } 636 637 if (Subtarget->hasSSE1()) { 638 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 639 640 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 641 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 642 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 643 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 644 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 645 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 646 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 647 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 648 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 649 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 650 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 651 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 652 } 653 654 if (Subtarget->hasSSE2()) { 655 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 656 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 657 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 658 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 659 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 660 661 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 662 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 663 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 664 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 665 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 666 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 667 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 668 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 669 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 670 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 671 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 672 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 673 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 674 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 675 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 676 677 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 678 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 679 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 680 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 681 682 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 683 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 684 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 685 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 686 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 687 688 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 689 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 690 MVT VT = (MVT::SimpleValueType)i; 691 // Do not attempt to custom lower non-power-of-2 vectors 692 if (!isPowerOf2_32(VT.getVectorNumElements())) 693 continue; 694 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 695 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 696 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 697 } 698 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 699 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 700 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 701 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 702 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 703 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 704 if (Subtarget->is64Bit()) { 705 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 706 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 707 } 708 709 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 710 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 711 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 712 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 713 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 714 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 715 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 716 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 717 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 718 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 719 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 720 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 721 } 722 723 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 724 725 // Custom lower v2i64 and v2f64 selects. 726 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 727 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 728 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 729 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 730 731 } 732 733 if (Subtarget->hasSSE41()) { 734 // FIXME: Do we need to handle scalar-to-vector here? 735 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 736 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 737 738 // i8 and i16 vectors are custom , because the source register and source 739 // source memory operand types are not the same width. f32 vectors are 740 // custom since the immediate controlling the insert encodes additional 741 // information. 742 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 743 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 744 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 745 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 746 747 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 748 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 749 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 750 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 751 752 if (Subtarget->is64Bit()) { 753 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 754 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 755 } 756 } 757 758 if (Subtarget->hasSSE42()) { 759 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 760 } 761 762 // We want to custom lower some of our intrinsics. 763 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 764 765 // We have target-specific dag combine patterns for the following nodes: 766 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 767 setTargetDAGCombine(ISD::BUILD_VECTOR); 768 setTargetDAGCombine(ISD::SELECT); 769 setTargetDAGCombine(ISD::STORE); 770 771 computeRegisterProperties(); 772 773 // FIXME: These should be based on subtarget info. Plus, the values should 774 // be smaller when we are in optimizing for size mode. 775 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 776 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 777 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 778 allowUnalignedMemoryAccesses = true; // x86 supports it! 779 setPrefLoopAlignment(16); 780} 781 782 783MVT X86TargetLowering::getSetCCResultType(const SDValue &) const { 784 return MVT::i8; 785} 786 787 788/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 789/// the desired ByVal argument alignment. 790static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 791 if (MaxAlign == 16) 792 return; 793 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 794 if (VTy->getBitWidth() == 128) 795 MaxAlign = 16; 796 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 797 unsigned EltAlign = 0; 798 getMaxByValAlign(ATy->getElementType(), EltAlign); 799 if (EltAlign > MaxAlign) 800 MaxAlign = EltAlign; 801 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 802 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 803 unsigned EltAlign = 0; 804 getMaxByValAlign(STy->getElementType(i), EltAlign); 805 if (EltAlign > MaxAlign) 806 MaxAlign = EltAlign; 807 if (MaxAlign == 16) 808 break; 809 } 810 } 811 return; 812} 813 814/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 815/// function arguments in the caller parameter area. For X86, aggregates 816/// that contain SSE vectors are placed at 16-byte boundaries while the rest 817/// are at 4-byte boundaries. 818unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 819 if (Subtarget->is64Bit()) { 820 // Max of 8 and alignment of type. 821 unsigned TyAlign = TD->getABITypeAlignment(Ty); 822 if (TyAlign > 8) 823 return TyAlign; 824 return 8; 825 } 826 827 unsigned Align = 4; 828 if (Subtarget->hasSSE1()) 829 getMaxByValAlign(Ty, Align); 830 return Align; 831} 832 833/// getOptimalMemOpType - Returns the target specific optimal type for load 834/// and store operations as a result of memset, memcpy, and memmove 835/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 836/// determining it. 837MVT 838X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 839 bool isSrcConst, bool isSrcStr) const { 840 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 841 return MVT::v4i32; 842 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 843 return MVT::v4f32; 844 if (Subtarget->is64Bit() && Size >= 8) 845 return MVT::i64; 846 return MVT::i32; 847} 848 849 850/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 851/// jumptable. 852SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 853 SelectionDAG &DAG) const { 854 if (usesGlobalOffsetTable()) 855 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 856 if (!Subtarget->isPICStyleRIPRel()) 857 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 858 return Table; 859} 860 861//===----------------------------------------------------------------------===// 862// Return Value Calling Convention Implementation 863//===----------------------------------------------------------------------===// 864 865#include "X86GenCallingConv.inc" 866 867/// LowerRET - Lower an ISD::RET node. 868SDValue X86TargetLowering::LowerRET(SDValue Op, SelectionDAG &DAG) { 869 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 870 871 SmallVector<CCValAssign, 16> RVLocs; 872 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 873 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 874 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 875 CCInfo.AnalyzeReturn(Op.getNode(), RetCC_X86); 876 877 // If this is the first return lowered for this function, add the regs to the 878 // liveout set for the function. 879 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 880 for (unsigned i = 0; i != RVLocs.size(); ++i) 881 if (RVLocs[i].isRegLoc()) 882 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 883 } 884 SDValue Chain = Op.getOperand(0); 885 886 // Handle tail call return. 887 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 888 if (Chain.getOpcode() == X86ISD::TAILCALL) { 889 SDValue TailCall = Chain; 890 SDValue TargetAddress = TailCall.getOperand(1); 891 SDValue StackAdjustment = TailCall.getOperand(2); 892 assert(((TargetAddress.getOpcode() == ISD::Register && 893 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 894 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 895 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 896 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 897 "Expecting an global address, external symbol, or register"); 898 assert(StackAdjustment.getOpcode() == ISD::Constant && 899 "Expecting a const value"); 900 901 SmallVector<SDValue,8> Operands; 902 Operands.push_back(Chain.getOperand(0)); 903 Operands.push_back(TargetAddress); 904 Operands.push_back(StackAdjustment); 905 // Copy registers used by the call. Last operand is a flag so it is not 906 // copied. 907 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 908 Operands.push_back(Chain.getOperand(i)); 909 } 910 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 911 Operands.size()); 912 } 913 914 // Regular return. 915 SDValue Flag; 916 917 SmallVector<SDValue, 6> RetOps; 918 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 919 // Operand #1 = Bytes To Pop 920 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 921 922 // Copy the result values into the output registers. 923 for (unsigned i = 0; i != RVLocs.size(); ++i) { 924 CCValAssign &VA = RVLocs[i]; 925 assert(VA.isRegLoc() && "Can only return in registers!"); 926 SDValue ValToCopy = Op.getOperand(i*2+1); 927 928 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 929 // the RET instruction and handled by the FP Stackifier. 930 if (RVLocs[i].getLocReg() == X86::ST0 || 931 RVLocs[i].getLocReg() == X86::ST1) { 932 // If this is a copy from an xmm register to ST(0), use an FPExtend to 933 // change the value to the FP stack register class. 934 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 935 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 936 RetOps.push_back(ValToCopy); 937 // Don't emit a copytoreg. 938 continue; 939 } 940 941 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 942 Flag = Chain.getValue(1); 943 } 944 945 // The x86-64 ABI for returning structs by value requires that we copy 946 // the sret argument into %rax for the return. We saved the argument into 947 // a virtual register in the entry block, so now we copy the value out 948 // and into %rax. 949 if (Subtarget->is64Bit() && 950 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 951 MachineFunction &MF = DAG.getMachineFunction(); 952 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 953 unsigned Reg = FuncInfo->getSRetReturnReg(); 954 if (!Reg) { 955 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 956 FuncInfo->setSRetReturnReg(Reg); 957 } 958 SDValue Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 959 960 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 961 Flag = Chain.getValue(1); 962 } 963 964 RetOps[0] = Chain; // Update chain. 965 966 // Add the flag if we have it. 967 if (Flag.getNode()) 968 RetOps.push_back(Flag); 969 970 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 971} 972 973 974/// LowerCallResult - Lower the result values of an ISD::CALL into the 975/// appropriate copies out of appropriate physical registers. This assumes that 976/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 977/// being lowered. The returns a SDNode with the same number of values as the 978/// ISD::CALL. 979SDNode *X86TargetLowering:: 980LowerCallResult(SDValue Chain, SDValue InFlag, SDNode *TheCall, 981 unsigned CallingConv, SelectionDAG &DAG) { 982 983 // Assign locations to each value returned by this call. 984 SmallVector<CCValAssign, 16> RVLocs; 985 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 986 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 987 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 988 989 SmallVector<SDValue, 8> ResultVals; 990 991 // Copy all of the result registers out of their specified physreg. 992 for (unsigned i = 0; i != RVLocs.size(); ++i) { 993 MVT CopyVT = RVLocs[i].getValVT(); 994 995 // If this is a call to a function that returns an fp value on the floating 996 // point stack, but where we prefer to use the value in xmm registers, copy 997 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 998 if ((RVLocs[i].getLocReg() == X86::ST0 || 999 RVLocs[i].getLocReg() == X86::ST1) && 1000 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 1001 CopyVT = MVT::f80; 1002 } 1003 1004 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 1005 CopyVT, InFlag).getValue(1); 1006 SDValue Val = Chain.getValue(0); 1007 InFlag = Chain.getValue(2); 1008 1009 if (CopyVT != RVLocs[i].getValVT()) { 1010 // Round the F80 the right size, which also moves to the appropriate xmm 1011 // register. 1012 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 1013 // This truncation won't change the value. 1014 DAG.getIntPtrConstant(1)); 1015 } 1016 1017 ResultVals.push_back(Val); 1018 } 1019 1020 // Merge everything together with a MERGE_VALUES node. 1021 ResultVals.push_back(Chain); 1022 return DAG.getMergeValues(TheCall->getVTList(), &ResultVals[0], 1023 ResultVals.size()).getNode(); 1024} 1025 1026 1027//===----------------------------------------------------------------------===// 1028// C & StdCall & Fast Calling Convention implementation 1029//===----------------------------------------------------------------------===// 1030// StdCall calling convention seems to be standard for many Windows' API 1031// routines and around. It differs from C calling convention just a little: 1032// callee should clean up the stack, not caller. Symbols should be also 1033// decorated in some fancy way :) It doesn't support any vector arguments. 1034// For info on fast calling convention see Fast Calling Convention (tail call) 1035// implementation LowerX86_32FastCCCallTo. 1036 1037/// AddLiveIn - This helper function adds the specified physical register to the 1038/// MachineFunction as a live in value. It also creates a corresponding virtual 1039/// register for it. 1040static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1041 const TargetRegisterClass *RC) { 1042 assert(RC->contains(PReg) && "Not the correct regclass!"); 1043 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1044 MF.getRegInfo().addLiveIn(PReg, VReg); 1045 return VReg; 1046} 1047 1048/// CallIsStructReturn - Determines whether a CALL node uses struct return 1049/// semantics. 1050static bool CallIsStructReturn(SDValue Op) { 1051 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1052 if (!NumOps) 1053 return false; 1054 1055 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1056} 1057 1058/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1059/// return semantics. 1060static bool ArgsAreStructReturn(SDValue Op) { 1061 unsigned NumArgs = Op.getNode()->getNumValues() - 1; 1062 if (!NumArgs) 1063 return false; 1064 1065 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1066} 1067 1068/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1069/// the callee to pop its own arguments. Callee pop is necessary to support tail 1070/// calls. 1071bool X86TargetLowering::IsCalleePop(SDValue Op) { 1072 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1073 if (IsVarArg) 1074 return false; 1075 1076 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1077 default: 1078 return false; 1079 case CallingConv::X86_StdCall: 1080 return !Subtarget->is64Bit(); 1081 case CallingConv::X86_FastCall: 1082 return !Subtarget->is64Bit(); 1083 case CallingConv::Fast: 1084 return PerformTailCallOpt; 1085 } 1086} 1087 1088/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1089/// FORMAL_ARGUMENTS node. 1090CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDValue Op) const { 1091 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1092 1093 if (Subtarget->is64Bit()) { 1094 if (Subtarget->isTargetWin64()) 1095 return CC_X86_Win64_C; 1096 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1097 return CC_X86_64_TailCall; 1098 else 1099 return CC_X86_64_C; 1100 } 1101 1102 if (CC == CallingConv::X86_FastCall) 1103 return CC_X86_32_FastCall; 1104 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1105 return CC_X86_32_TailCall; 1106 else if (CC == CallingConv::Fast) 1107 return CC_X86_32_FastCC; 1108 else 1109 return CC_X86_32_C; 1110} 1111 1112/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1113/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1114NameDecorationStyle 1115X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDValue Op) { 1116 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1117 if (CC == CallingConv::X86_FastCall) 1118 return FastCall; 1119 else if (CC == CallingConv::X86_StdCall) 1120 return StdCall; 1121 return None; 1122} 1123 1124 1125/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1126/// in a register before calling. 1127bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1128 return !IsTailCall && !Is64Bit && 1129 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1130 Subtarget->isPICStyleGOT(); 1131} 1132 1133/// CallRequiresFnAddressInReg - Check whether the call requires the function 1134/// address to be loaded in a register. 1135bool 1136X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1137 return !Is64Bit && IsTailCall && 1138 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1139 Subtarget->isPICStyleGOT(); 1140} 1141 1142/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1143/// by "Src" to address "Dst" with size and alignment information specified by 1144/// the specific parameter attribute. The copy will be passed as a byval 1145/// function parameter. 1146static SDValue 1147CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 1148 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1149 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1150 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1151 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1152} 1153 1154SDValue X86TargetLowering::LowerMemArgument(SDValue Op, SelectionDAG &DAG, 1155 const CCValAssign &VA, 1156 MachineFrameInfo *MFI, 1157 unsigned CC, 1158 SDValue Root, unsigned i) { 1159 // Create the nodes corresponding to a load from this parameter slot. 1160 ISD::ArgFlagsTy Flags = 1161 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1162 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1163 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1164 1165 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1166 // changed with more analysis. 1167 // In case of tail call optimization mark all arguments mutable. Since they 1168 // could be overwritten by lowering of arguments in case of a tail call. 1169 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1170 VA.getLocMemOffset(), isImmutable); 1171 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 1172 if (Flags.isByVal()) 1173 return FIN; 1174 return DAG.getLoad(VA.getValVT(), Root, FIN, 1175 PseudoSourceValue::getFixedStack(FI), 0); 1176} 1177 1178SDValue 1179X86TargetLowering::LowerFORMAL_ARGUMENTS(SDValue Op, SelectionDAG &DAG) { 1180 MachineFunction &MF = DAG.getMachineFunction(); 1181 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1182 1183 const Function* Fn = MF.getFunction(); 1184 if (Fn->hasExternalLinkage() && 1185 Subtarget->isTargetCygMing() && 1186 Fn->getName() == "main") 1187 FuncInfo->setForceFramePointer(true); 1188 1189 // Decorate the function name. 1190 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1191 1192 MachineFrameInfo *MFI = MF.getFrameInfo(); 1193 SDValue Root = Op.getOperand(0); 1194 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1195 unsigned CC = MF.getFunction()->getCallingConv(); 1196 bool Is64Bit = Subtarget->is64Bit(); 1197 bool IsWin64 = Subtarget->isTargetWin64(); 1198 1199 assert(!(isVarArg && CC == CallingConv::Fast) && 1200 "Var args not supported with calling convention fastcc"); 1201 1202 // Assign locations to all of the incoming arguments. 1203 SmallVector<CCValAssign, 16> ArgLocs; 1204 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1205 CCInfo.AnalyzeFormalArguments(Op.getNode(), CCAssignFnForNode(Op)); 1206 1207 SmallVector<SDValue, 8> ArgValues; 1208 unsigned LastVal = ~0U; 1209 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1210 CCValAssign &VA = ArgLocs[i]; 1211 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1212 // places. 1213 assert(VA.getValNo() != LastVal && 1214 "Don't support value assigned to multiple locs yet"); 1215 LastVal = VA.getValNo(); 1216 1217 if (VA.isRegLoc()) { 1218 MVT RegVT = VA.getLocVT(); 1219 TargetRegisterClass *RC; 1220 if (RegVT == MVT::i32) 1221 RC = X86::GR32RegisterClass; 1222 else if (Is64Bit && RegVT == MVT::i64) 1223 RC = X86::GR64RegisterClass; 1224 else if (RegVT == MVT::f32) 1225 RC = X86::FR32RegisterClass; 1226 else if (RegVT == MVT::f64) 1227 RC = X86::FR64RegisterClass; 1228 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1229 RC = X86::VR128RegisterClass; 1230 else if (RegVT.isVector()) { 1231 assert(RegVT.getSizeInBits() == 64); 1232 if (!Is64Bit) 1233 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1234 else { 1235 // Darwin calling convention passes MMX values in either GPRs or 1236 // XMMs in x86-64. Other targets pass them in memory. 1237 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1238 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1239 RegVT = MVT::v2i64; 1240 } else { 1241 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1242 RegVT = MVT::i64; 1243 } 1244 } 1245 } else { 1246 assert(0 && "Unknown argument type!"); 1247 } 1248 1249 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1250 SDValue ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1251 1252 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1253 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1254 // right size. 1255 if (VA.getLocInfo() == CCValAssign::SExt) 1256 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1257 DAG.getValueType(VA.getValVT())); 1258 else if (VA.getLocInfo() == CCValAssign::ZExt) 1259 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1260 DAG.getValueType(VA.getValVT())); 1261 1262 if (VA.getLocInfo() != CCValAssign::Full) 1263 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1264 1265 // Handle MMX values passed in GPRs. 1266 if (Is64Bit && RegVT != VA.getLocVT()) { 1267 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1268 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1269 else if (RC == X86::VR128RegisterClass) { 1270 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1271 DAG.getConstant(0, MVT::i64)); 1272 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1273 } 1274 } 1275 1276 ArgValues.push_back(ArgValue); 1277 } else { 1278 assert(VA.isMemLoc()); 1279 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1280 } 1281 } 1282 1283 // The x86-64 ABI for returning structs by value requires that we copy 1284 // the sret argument into %rax for the return. Save the argument into 1285 // a virtual register so that we can access it from the return points. 1286 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1287 MachineFunction &MF = DAG.getMachineFunction(); 1288 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1289 unsigned Reg = FuncInfo->getSRetReturnReg(); 1290 if (!Reg) { 1291 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1292 FuncInfo->setSRetReturnReg(Reg); 1293 } 1294 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1295 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1296 } 1297 1298 unsigned StackSize = CCInfo.getNextStackOffset(); 1299 // align stack specially for tail calls 1300 if (PerformTailCallOpt && CC == CallingConv::Fast) 1301 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1302 1303 // If the function takes variable number of arguments, make a frame index for 1304 // the start of the first vararg value... for expansion of llvm.va_start. 1305 if (isVarArg) { 1306 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1307 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1308 } 1309 if (Is64Bit) { 1310 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1311 1312 // FIXME: We should really autogenerate these arrays 1313 static const unsigned GPR64ArgRegsWin64[] = { 1314 X86::RCX, X86::RDX, X86::R8, X86::R9 1315 }; 1316 static const unsigned XMMArgRegsWin64[] = { 1317 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1318 }; 1319 static const unsigned GPR64ArgRegs64Bit[] = { 1320 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1321 }; 1322 static const unsigned XMMArgRegs64Bit[] = { 1323 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1324 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1325 }; 1326 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1327 1328 if (IsWin64) { 1329 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1330 GPR64ArgRegs = GPR64ArgRegsWin64; 1331 XMMArgRegs = XMMArgRegsWin64; 1332 } else { 1333 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1334 GPR64ArgRegs = GPR64ArgRegs64Bit; 1335 XMMArgRegs = XMMArgRegs64Bit; 1336 } 1337 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1338 TotalNumIntRegs); 1339 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1340 TotalNumXMMRegs); 1341 1342 // For X86-64, if there are vararg parameters that are passed via 1343 // registers, then we must store them to their spots on the stack so they 1344 // may be loaded by deferencing the result of va_next. 1345 VarArgsGPOffset = NumIntRegs * 8; 1346 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1347 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1348 TotalNumXMMRegs * 16, 16); 1349 1350 // Store the integer parameter registers. 1351 SmallVector<SDValue, 8> MemOps; 1352 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1353 SDValue FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1354 DAG.getIntPtrConstant(VarArgsGPOffset)); 1355 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1356 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1357 X86::GR64RegisterClass); 1358 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1359 SDValue Store = 1360 DAG.getStore(Val.getValue(1), Val, FIN, 1361 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1362 MemOps.push_back(Store); 1363 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1364 DAG.getIntPtrConstant(8)); 1365 } 1366 1367 // Now store the XMM (fp + vector) parameter registers. 1368 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1369 DAG.getIntPtrConstant(VarArgsFPOffset)); 1370 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1371 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1372 X86::VR128RegisterClass); 1373 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1374 SDValue Store = 1375 DAG.getStore(Val.getValue(1), Val, FIN, 1376 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1377 MemOps.push_back(Store); 1378 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1379 DAG.getIntPtrConstant(16)); 1380 } 1381 if (!MemOps.empty()) 1382 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1383 &MemOps[0], MemOps.size()); 1384 } 1385 } 1386 1387 ArgValues.push_back(Root); 1388 1389 // Some CCs need callee pop. 1390 if (IsCalleePop(Op)) { 1391 BytesToPopOnReturn = StackSize; // Callee pops everything. 1392 BytesCallerReserves = 0; 1393 } else { 1394 BytesToPopOnReturn = 0; // Callee pops nothing. 1395 // If this is an sret function, the return should pop the hidden pointer. 1396 if (!Is64Bit && CC != CallingConv::Fast && ArgsAreStructReturn(Op)) 1397 BytesToPopOnReturn = 4; 1398 BytesCallerReserves = StackSize; 1399 } 1400 1401 if (!Is64Bit) { 1402 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1403 if (CC == CallingConv::X86_FastCall) 1404 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1405 } 1406 1407 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1408 1409 // Return the new list of results. 1410 return DAG.getMergeValues(Op.getNode()->getVTList(), &ArgValues[0], 1411 ArgValues.size()).getValue(Op.getResNo()); 1412} 1413 1414SDValue 1415X86TargetLowering::LowerMemOpCallTo(SDValue Op, SelectionDAG &DAG, 1416 const SDValue &StackPtr, 1417 const CCValAssign &VA, 1418 SDValue Chain, 1419 SDValue Arg) { 1420 unsigned LocMemOffset = VA.getLocMemOffset(); 1421 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1422 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1423 ISD::ArgFlagsTy Flags = 1424 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1425 if (Flags.isByVal()) { 1426 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1427 } 1428 return DAG.getStore(Chain, Arg, PtrOff, 1429 PseudoSourceValue::getStack(), LocMemOffset); 1430} 1431 1432/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1433/// optimization is performed and it is required. 1434SDValue 1435X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1436 SDValue &OutRetAddr, 1437 SDValue Chain, 1438 bool IsTailCall, 1439 bool Is64Bit, 1440 int FPDiff) { 1441 if (!IsTailCall || FPDiff==0) return Chain; 1442 1443 // Adjust the Return address stack slot. 1444 MVT VT = getPointerTy(); 1445 OutRetAddr = getReturnAddressFrameIndex(DAG); 1446 // Load the "old" Return address. 1447 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1448 return SDValue(OutRetAddr.getNode(), 1); 1449} 1450 1451/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1452/// optimization is performed and it is required (FPDiff!=0). 1453static SDValue 1454EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1455 SDValue Chain, SDValue RetAddrFrIdx, 1456 bool Is64Bit, int FPDiff) { 1457 // Store the return address to the appropriate stack slot. 1458 if (!FPDiff) return Chain; 1459 // Calculate the new stack slot for the return address. 1460 int SlotSize = Is64Bit ? 8 : 4; 1461 int NewReturnAddrFI = 1462 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1463 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1464 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1465 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1466 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 1467 return Chain; 1468} 1469 1470SDValue X86TargetLowering::LowerCALL(SDValue Op, SelectionDAG &DAG) { 1471 MachineFunction &MF = DAG.getMachineFunction(); 1472 SDValue Chain = Op.getOperand(0); 1473 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1474 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1475 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1476 && CC == CallingConv::Fast && PerformTailCallOpt; 1477 SDValue Callee = Op.getOperand(4); 1478 bool Is64Bit = Subtarget->is64Bit(); 1479 bool IsStructRet = CallIsStructReturn(Op); 1480 1481 assert(!(isVarArg && CC == CallingConv::Fast) && 1482 "Var args not supported with calling convention fastcc"); 1483 1484 // Analyze operands of the call, assigning locations to each operand. 1485 SmallVector<CCValAssign, 16> ArgLocs; 1486 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1487 CCInfo.AnalyzeCallOperands(Op.getNode(), CCAssignFnForNode(Op)); 1488 1489 // Get a count of how many bytes are to be pushed on the stack. 1490 unsigned NumBytes = CCInfo.getNextStackOffset(); 1491 if (PerformTailCallOpt && CC == CallingConv::Fast) 1492 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1493 1494 int FPDiff = 0; 1495 if (IsTailCall) { 1496 // Lower arguments at fp - stackoffset + fpdiff. 1497 unsigned NumBytesCallerPushed = 1498 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1499 FPDiff = NumBytesCallerPushed - NumBytes; 1500 1501 // Set the delta of movement of the returnaddr stackslot. 1502 // But only set if delta is greater than previous delta. 1503 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1504 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1505 } 1506 1507 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1508 1509 SDValue RetAddrFrIdx; 1510 // Load return adress for tail calls. 1511 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1512 FPDiff); 1513 1514 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 1515 SmallVector<SDValue, 8> MemOpChains; 1516 SDValue StackPtr; 1517 1518 // Walk the register/memloc assignments, inserting copies/loads. In the case 1519 // of tail call optimization arguments are handle later. 1520 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1521 CCValAssign &VA = ArgLocs[i]; 1522 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1523 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1524 getArgFlags().isByVal(); 1525 1526 // Promote the value if needed. 1527 switch (VA.getLocInfo()) { 1528 default: assert(0 && "Unknown loc info!"); 1529 case CCValAssign::Full: break; 1530 case CCValAssign::SExt: 1531 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1532 break; 1533 case CCValAssign::ZExt: 1534 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1535 break; 1536 case CCValAssign::AExt: 1537 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1538 break; 1539 } 1540 1541 if (VA.isRegLoc()) { 1542 if (Is64Bit) { 1543 MVT RegVT = VA.getLocVT(); 1544 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1545 switch (VA.getLocReg()) { 1546 default: 1547 break; 1548 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1549 case X86::R8: { 1550 // Special case: passing MMX values in GPR registers. 1551 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1552 break; 1553 } 1554 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1555 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1556 // Special case: passing MMX values in XMM registers. 1557 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1558 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1559 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1560 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1561 getMOVLMask(2, DAG)); 1562 break; 1563 } 1564 } 1565 } 1566 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1567 } else { 1568 if (!IsTailCall || (IsTailCall && isByVal)) { 1569 assert(VA.isMemLoc()); 1570 if (StackPtr.getNode() == 0) 1571 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1572 1573 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1574 Arg)); 1575 } 1576 } 1577 } 1578 1579 if (!MemOpChains.empty()) 1580 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1581 &MemOpChains[0], MemOpChains.size()); 1582 1583 // Build a sequence of copy-to-reg nodes chained together with token chain 1584 // and flag operands which copy the outgoing args into registers. 1585 SDValue InFlag; 1586 // Tail call byval lowering might overwrite argument registers so in case of 1587 // tail call optimization the copies to registers are lowered later. 1588 if (!IsTailCall) 1589 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1590 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1591 InFlag); 1592 InFlag = Chain.getValue(1); 1593 } 1594 1595 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1596 // GOT pointer. 1597 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1598 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1599 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1600 InFlag); 1601 InFlag = Chain.getValue(1); 1602 } 1603 // If we are tail calling and generating PIC/GOT style code load the address 1604 // of the callee into ecx. The value in ecx is used as target of the tail 1605 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1606 // calls on PIC/GOT architectures. Normally we would just put the address of 1607 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1608 // restored (since ebx is callee saved) before jumping to the target@PLT. 1609 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1610 // Note: The actual moving to ecx is done further down. 1611 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1612 if (G && !G->getGlobal()->hasHiddenVisibility() && 1613 !G->getGlobal()->hasProtectedVisibility()) 1614 Callee = LowerGlobalAddress(Callee, DAG); 1615 else if (isa<ExternalSymbolSDNode>(Callee)) 1616 Callee = LowerExternalSymbol(Callee,DAG); 1617 } 1618 1619 if (Is64Bit && isVarArg) { 1620 // From AMD64 ABI document: 1621 // For calls that may call functions that use varargs or stdargs 1622 // (prototype-less calls or calls to functions containing ellipsis (...) in 1623 // the declaration) %al is used as hidden argument to specify the number 1624 // of SSE registers used. The contents of %al do not need to match exactly 1625 // the number of registers, but must be an ubound on the number of SSE 1626 // registers used and is in the range 0 - 8 inclusive. 1627 1628 // FIXME: Verify this on Win64 1629 // Count the number of XMM registers allocated. 1630 static const unsigned XMMArgRegs[] = { 1631 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1632 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1633 }; 1634 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1635 1636 Chain = DAG.getCopyToReg(Chain, X86::AL, 1637 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1638 InFlag = Chain.getValue(1); 1639 } 1640 1641 1642 // For tail calls lower the arguments to the 'real' stack slot. 1643 if (IsTailCall) { 1644 SmallVector<SDValue, 8> MemOpChains2; 1645 SDValue FIN; 1646 int FI = 0; 1647 // Do not flag preceeding copytoreg stuff together with the following stuff. 1648 InFlag = SDValue(); 1649 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1650 CCValAssign &VA = ArgLocs[i]; 1651 if (!VA.isRegLoc()) { 1652 assert(VA.isMemLoc()); 1653 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1654 SDValue FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1655 ISD::ArgFlagsTy Flags = 1656 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1657 // Create frame index. 1658 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1659 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1660 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1661 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1662 1663 if (Flags.isByVal()) { 1664 // Copy relative to framepointer. 1665 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1666 if (StackPtr.getNode() == 0) 1667 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1668 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1669 1670 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1671 Flags, DAG)); 1672 } else { 1673 // Store relative to framepointer. 1674 MemOpChains2.push_back( 1675 DAG.getStore(Chain, Arg, FIN, 1676 PseudoSourceValue::getFixedStack(FI), 0)); 1677 } 1678 } 1679 } 1680 1681 if (!MemOpChains2.empty()) 1682 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1683 &MemOpChains2[0], MemOpChains2.size()); 1684 1685 // Copy arguments to their registers. 1686 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1687 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1688 InFlag); 1689 InFlag = Chain.getValue(1); 1690 } 1691 InFlag =SDValue(); 1692 1693 // Store the return address to the appropriate stack slot. 1694 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1695 FPDiff); 1696 } 1697 1698 // If the callee is a GlobalAddress node (quite common, every direct call is) 1699 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1700 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1701 // We should use extra load for direct calls to dllimported functions in 1702 // non-JIT mode. 1703 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1704 getTargetMachine(), true)) 1705 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1706 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1707 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1708 } else if (IsTailCall) { 1709 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1710 1711 Chain = DAG.getCopyToReg(Chain, 1712 DAG.getRegister(Opc, getPointerTy()), 1713 Callee,InFlag); 1714 Callee = DAG.getRegister(Opc, getPointerTy()); 1715 // Add register as live out. 1716 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1717 } 1718 1719 // Returns a chain & a flag for retval copy to use. 1720 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1721 SmallVector<SDValue, 8> Ops; 1722 1723 if (IsTailCall) { 1724 Ops.push_back(Chain); 1725 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1726 Ops.push_back(DAG.getIntPtrConstant(0)); 1727 if (InFlag.getNode()) 1728 Ops.push_back(InFlag); 1729 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1730 InFlag = Chain.getValue(1); 1731 1732 // Returns a chain & a flag for retval copy to use. 1733 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1734 Ops.clear(); 1735 } 1736 1737 Ops.push_back(Chain); 1738 Ops.push_back(Callee); 1739 1740 if (IsTailCall) 1741 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1742 1743 // Add argument registers to the end of the list so that they are known live 1744 // into the call. 1745 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1746 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1747 RegsToPass[i].second.getValueType())); 1748 1749 // Add an implicit use GOT pointer in EBX. 1750 if (!IsTailCall && !Is64Bit && 1751 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1752 Subtarget->isPICStyleGOT()) 1753 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1754 1755 // Add an implicit use of AL for x86 vararg functions. 1756 if (Is64Bit && isVarArg) 1757 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1758 1759 if (InFlag.getNode()) 1760 Ops.push_back(InFlag); 1761 1762 if (IsTailCall) { 1763 assert(InFlag.getNode() && 1764 "Flag must be set. Depend on flag being set in LowerRET"); 1765 Chain = DAG.getNode(X86ISD::TAILCALL, 1766 Op.getNode()->getVTList(), &Ops[0], Ops.size()); 1767 1768 return SDValue(Chain.getNode(), Op.getResNo()); 1769 } 1770 1771 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1772 InFlag = Chain.getValue(1); 1773 1774 // Create the CALLSEQ_END node. 1775 unsigned NumBytesForCalleeToPush; 1776 if (IsCalleePop(Op)) 1777 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1778 else if (!Is64Bit && CC != CallingConv::Fast && IsStructRet) 1779 // If this is is a call to a struct-return function, the callee 1780 // pops the hidden struct pointer, so we have to push it back. 1781 // This is common for Darwin/X86, Linux & Mingw32 targets. 1782 NumBytesForCalleeToPush = 4; 1783 else 1784 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1785 1786 // Returns a flag for retval copy to use. 1787 Chain = DAG.getCALLSEQ_END(Chain, 1788 DAG.getIntPtrConstant(NumBytes), 1789 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1790 InFlag); 1791 InFlag = Chain.getValue(1); 1792 1793 // Handle result values, copying them out of physregs into vregs that we 1794 // return. 1795 return SDValue(LowerCallResult(Chain, InFlag, Op.getNode(), CC, DAG), 1796 Op.getResNo()); 1797} 1798 1799 1800//===----------------------------------------------------------------------===// 1801// Fast Calling Convention (tail call) implementation 1802//===----------------------------------------------------------------------===// 1803 1804// Like std call, callee cleans arguments, convention except that ECX is 1805// reserved for storing the tail called function address. Only 2 registers are 1806// free for argument passing (inreg). Tail call optimization is performed 1807// provided: 1808// * tailcallopt is enabled 1809// * caller/callee are fastcc 1810// On X86_64 architecture with GOT-style position independent code only local 1811// (within module) calls are supported at the moment. 1812// To keep the stack aligned according to platform abi the function 1813// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1814// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1815// If a tail called function callee has more arguments than the caller the 1816// caller needs to make sure that there is room to move the RETADDR to. This is 1817// achieved by reserving an area the size of the argument delta right after the 1818// original REtADDR, but before the saved framepointer or the spilled registers 1819// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1820// stack layout: 1821// arg1 1822// arg2 1823// RETADDR 1824// [ new RETADDR 1825// move area ] 1826// (possible EBP) 1827// ESI 1828// EDI 1829// local1 .. 1830 1831/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1832/// for a 16 byte align requirement. 1833unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1834 SelectionDAG& DAG) { 1835 MachineFunction &MF = DAG.getMachineFunction(); 1836 const TargetMachine &TM = MF.getTarget(); 1837 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1838 unsigned StackAlignment = TFI.getStackAlignment(); 1839 uint64_t AlignMask = StackAlignment - 1; 1840 int64_t Offset = StackSize; 1841 uint64_t SlotSize = TD->getPointerSize(); 1842 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1843 // Number smaller than 12 so just add the difference. 1844 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1845 } else { 1846 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1847 Offset = ((~AlignMask) & Offset) + StackAlignment + 1848 (StackAlignment-SlotSize); 1849 } 1850 return Offset; 1851} 1852 1853/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1854/// following the call is a return. A function is eligible if caller/callee 1855/// calling conventions match, currently only fastcc supports tail calls, and 1856/// the function CALL is immediatly followed by a RET. 1857bool X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Call, 1858 SDValue Ret, 1859 SelectionDAG& DAG) const { 1860 if (!PerformTailCallOpt) 1861 return false; 1862 1863 if (CheckTailCallReturnConstraints(Call, Ret)) { 1864 MachineFunction &MF = DAG.getMachineFunction(); 1865 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1866 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1867 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1868 SDValue Callee = Call.getOperand(4); 1869 // On x86/32Bit PIC/GOT tail calls are supported. 1870 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1871 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1872 return true; 1873 1874 // Can only do local tail calls (in same module, hidden or protected) on 1875 // x86_64 PIC/GOT at the moment. 1876 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1877 return G->getGlobal()->hasHiddenVisibility() 1878 || G->getGlobal()->hasProtectedVisibility(); 1879 } 1880 } 1881 1882 return false; 1883} 1884 1885FastISel * 1886X86TargetLowering::createFastISel(MachineFunction &mf, 1887 DenseMap<const Value *, unsigned> &vm, 1888 DenseMap<const BasicBlock *, 1889 MachineBasicBlock *> &bm, 1890 DenseMap<const AllocaInst *, int> &am) { 1891 1892 return X86::createFastISel(mf, vm, bm, am); 1893} 1894 1895 1896//===----------------------------------------------------------------------===// 1897// Other Lowering Hooks 1898//===----------------------------------------------------------------------===// 1899 1900 1901SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1902 MachineFunction &MF = DAG.getMachineFunction(); 1903 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1904 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1905 uint64_t SlotSize = TD->getPointerSize(); 1906 1907 if (ReturnAddrIndex == 0) { 1908 // Set up a frame object for the return address. 1909 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize); 1910 FuncInfo->setRAIndex(ReturnAddrIndex); 1911 } 1912 1913 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1914} 1915 1916 1917/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1918/// specific condition code. It returns a false if it cannot do a direct 1919/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1920/// needed. 1921static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1922 unsigned &X86CC, SDValue &LHS, SDValue &RHS, 1923 SelectionDAG &DAG) { 1924 X86CC = X86::COND_INVALID; 1925 if (!isFP) { 1926 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1927 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1928 // X > -1 -> X == 0, jump !sign. 1929 RHS = DAG.getConstant(0, RHS.getValueType()); 1930 X86CC = X86::COND_NS; 1931 return true; 1932 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1933 // X < 0 -> X == 0, jump on sign. 1934 X86CC = X86::COND_S; 1935 return true; 1936 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1937 // X < 1 -> X <= 0 1938 RHS = DAG.getConstant(0, RHS.getValueType()); 1939 X86CC = X86::COND_LE; 1940 return true; 1941 } 1942 } 1943 1944 switch (SetCCOpcode) { 1945 default: break; 1946 case ISD::SETEQ: X86CC = X86::COND_E; break; 1947 case ISD::SETGT: X86CC = X86::COND_G; break; 1948 case ISD::SETGE: X86CC = X86::COND_GE; break; 1949 case ISD::SETLT: X86CC = X86::COND_L; break; 1950 case ISD::SETLE: X86CC = X86::COND_LE; break; 1951 case ISD::SETNE: X86CC = X86::COND_NE; break; 1952 case ISD::SETULT: X86CC = X86::COND_B; break; 1953 case ISD::SETUGT: X86CC = X86::COND_A; break; 1954 case ISD::SETULE: X86CC = X86::COND_BE; break; 1955 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1956 } 1957 } else { 1958 // First determine if it requires or is profitable to flip the operands. 1959 bool Flip = false; 1960 switch (SetCCOpcode) { 1961 default: break; 1962 case ISD::SETOLT: 1963 case ISD::SETOLE: 1964 case ISD::SETUGT: 1965 case ISD::SETUGE: 1966 Flip = true; 1967 break; 1968 } 1969 1970 // If LHS is a foldable load, but RHS is not, flip the condition. 1971 if (!Flip && 1972 (ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) && 1973 !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) { 1974 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); 1975 Flip = true; 1976 } 1977 if (Flip) 1978 std::swap(LHS, RHS); 1979 1980 // On a floating point condition, the flags are set as follows: 1981 // ZF PF CF op 1982 // 0 | 0 | 0 | X > Y 1983 // 0 | 0 | 1 | X < Y 1984 // 1 | 0 | 0 | X == Y 1985 // 1 | 1 | 1 | unordered 1986 switch (SetCCOpcode) { 1987 default: break; 1988 case ISD::SETUEQ: 1989 case ISD::SETEQ: 1990 X86CC = X86::COND_E; 1991 break; 1992 case ISD::SETOLT: // flipped 1993 case ISD::SETOGT: 1994 case ISD::SETGT: 1995 X86CC = X86::COND_A; 1996 break; 1997 case ISD::SETOLE: // flipped 1998 case ISD::SETOGE: 1999 case ISD::SETGE: 2000 X86CC = X86::COND_AE; 2001 break; 2002 case ISD::SETUGT: // flipped 2003 case ISD::SETULT: 2004 case ISD::SETLT: 2005 X86CC = X86::COND_B; 2006 break; 2007 case ISD::SETUGE: // flipped 2008 case ISD::SETULE: 2009 case ISD::SETLE: 2010 X86CC = X86::COND_BE; 2011 break; 2012 case ISD::SETONE: 2013 case ISD::SETNE: 2014 X86CC = X86::COND_NE; 2015 break; 2016 case ISD::SETUO: 2017 X86CC = X86::COND_P; 2018 break; 2019 case ISD::SETO: 2020 X86CC = X86::COND_NP; 2021 break; 2022 } 2023 } 2024 2025 return X86CC != X86::COND_INVALID; 2026} 2027 2028/// hasFPCMov - is there a floating point cmov for the specific X86 condition 2029/// code. Current x86 isa includes the following FP cmov instructions: 2030/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 2031static bool hasFPCMov(unsigned X86CC) { 2032 switch (X86CC) { 2033 default: 2034 return false; 2035 case X86::COND_B: 2036 case X86::COND_BE: 2037 case X86::COND_E: 2038 case X86::COND_P: 2039 case X86::COND_A: 2040 case X86::COND_AE: 2041 case X86::COND_NE: 2042 case X86::COND_NP: 2043 return true; 2044 } 2045} 2046 2047/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 2048/// true if Op is undef or if its value falls within the specified range (L, H]. 2049static bool isUndefOrInRange(SDValue Op, unsigned Low, unsigned Hi) { 2050 if (Op.getOpcode() == ISD::UNDEF) 2051 return true; 2052 2053 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 2054 return (Val >= Low && Val < Hi); 2055} 2056 2057/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 2058/// true if Op is undef or if its value equal to the specified value. 2059static bool isUndefOrEqual(SDValue Op, unsigned Val) { 2060 if (Op.getOpcode() == ISD::UNDEF) 2061 return true; 2062 return cast<ConstantSDNode>(Op)->getValue() == Val; 2063} 2064 2065/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2066/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2067bool X86::isPSHUFDMask(SDNode *N) { 2068 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2069 2070 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2071 return false; 2072 2073 // Check if the value doesn't reference the second vector. 2074 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2075 SDValue Arg = N->getOperand(i); 2076 if (Arg.getOpcode() == ISD::UNDEF) continue; 2077 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2078 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2079 return false; 2080 } 2081 2082 return true; 2083} 2084 2085/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2086/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2087bool X86::isPSHUFHWMask(SDNode *N) { 2088 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2089 2090 if (N->getNumOperands() != 8) 2091 return false; 2092 2093 // Lower quadword copied in order. 2094 for (unsigned i = 0; i != 4; ++i) { 2095 SDValue Arg = N->getOperand(i); 2096 if (Arg.getOpcode() == ISD::UNDEF) continue; 2097 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2098 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2099 return false; 2100 } 2101 2102 // Upper quadword shuffled. 2103 for (unsigned i = 4; i != 8; ++i) { 2104 SDValue Arg = N->getOperand(i); 2105 if (Arg.getOpcode() == ISD::UNDEF) continue; 2106 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2107 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2108 if (Val < 4 || Val > 7) 2109 return false; 2110 } 2111 2112 return true; 2113} 2114 2115/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2116/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2117bool X86::isPSHUFLWMask(SDNode *N) { 2118 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2119 2120 if (N->getNumOperands() != 8) 2121 return false; 2122 2123 // Upper quadword copied in order. 2124 for (unsigned i = 4; i != 8; ++i) 2125 if (!isUndefOrEqual(N->getOperand(i), i)) 2126 return false; 2127 2128 // Lower quadword shuffled. 2129 for (unsigned i = 0; i != 4; ++i) 2130 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2131 return false; 2132 2133 return true; 2134} 2135 2136/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2137/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2138static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2139 if (NumElems != 2 && NumElems != 4) return false; 2140 2141 unsigned Half = NumElems / 2; 2142 for (unsigned i = 0; i < Half; ++i) 2143 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2144 return false; 2145 for (unsigned i = Half; i < NumElems; ++i) 2146 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2147 return false; 2148 2149 return true; 2150} 2151 2152bool X86::isSHUFPMask(SDNode *N) { 2153 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2154 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2155} 2156 2157/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2158/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2159/// half elements to come from vector 1 (which would equal the dest.) and 2160/// the upper half to come from vector 2. 2161static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2162 if (NumOps != 2 && NumOps != 4) return false; 2163 2164 unsigned Half = NumOps / 2; 2165 for (unsigned i = 0; i < Half; ++i) 2166 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2167 return false; 2168 for (unsigned i = Half; i < NumOps; ++i) 2169 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2170 return false; 2171 return true; 2172} 2173 2174static bool isCommutedSHUFP(SDNode *N) { 2175 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2176 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2177} 2178 2179/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2180/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2181bool X86::isMOVHLPSMask(SDNode *N) { 2182 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2183 2184 if (N->getNumOperands() != 4) 2185 return false; 2186 2187 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2188 return isUndefOrEqual(N->getOperand(0), 6) && 2189 isUndefOrEqual(N->getOperand(1), 7) && 2190 isUndefOrEqual(N->getOperand(2), 2) && 2191 isUndefOrEqual(N->getOperand(3), 3); 2192} 2193 2194/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2195/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2196/// <2, 3, 2, 3> 2197bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2198 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2199 2200 if (N->getNumOperands() != 4) 2201 return false; 2202 2203 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2204 return isUndefOrEqual(N->getOperand(0), 2) && 2205 isUndefOrEqual(N->getOperand(1), 3) && 2206 isUndefOrEqual(N->getOperand(2), 2) && 2207 isUndefOrEqual(N->getOperand(3), 3); 2208} 2209 2210/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2211/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2212bool X86::isMOVLPMask(SDNode *N) { 2213 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2214 2215 unsigned NumElems = N->getNumOperands(); 2216 if (NumElems != 2 && NumElems != 4) 2217 return false; 2218 2219 for (unsigned i = 0; i < NumElems/2; ++i) 2220 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2221 return false; 2222 2223 for (unsigned i = NumElems/2; i < NumElems; ++i) 2224 if (!isUndefOrEqual(N->getOperand(i), i)) 2225 return false; 2226 2227 return true; 2228} 2229 2230/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2231/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2232/// and MOVLHPS. 2233bool X86::isMOVHPMask(SDNode *N) { 2234 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2235 2236 unsigned NumElems = N->getNumOperands(); 2237 if (NumElems != 2 && NumElems != 4) 2238 return false; 2239 2240 for (unsigned i = 0; i < NumElems/2; ++i) 2241 if (!isUndefOrEqual(N->getOperand(i), i)) 2242 return false; 2243 2244 for (unsigned i = 0; i < NumElems/2; ++i) { 2245 SDValue Arg = N->getOperand(i + NumElems/2); 2246 if (!isUndefOrEqual(Arg, i + NumElems)) 2247 return false; 2248 } 2249 2250 return true; 2251} 2252 2253/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2254/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2255bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2256 bool V2IsSplat = false) { 2257 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2258 return false; 2259 2260 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2261 SDValue BitI = Elts[i]; 2262 SDValue BitI1 = Elts[i+1]; 2263 if (!isUndefOrEqual(BitI, j)) 2264 return false; 2265 if (V2IsSplat) { 2266 if (isUndefOrEqual(BitI1, NumElts)) 2267 return false; 2268 } else { 2269 if (!isUndefOrEqual(BitI1, j + NumElts)) 2270 return false; 2271 } 2272 } 2273 2274 return true; 2275} 2276 2277bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2278 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2279 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2280} 2281 2282/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2283/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2284bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2285 bool V2IsSplat = false) { 2286 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2287 return false; 2288 2289 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2290 SDValue BitI = Elts[i]; 2291 SDValue BitI1 = Elts[i+1]; 2292 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2293 return false; 2294 if (V2IsSplat) { 2295 if (isUndefOrEqual(BitI1, NumElts)) 2296 return false; 2297 } else { 2298 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2299 return false; 2300 } 2301 } 2302 2303 return true; 2304} 2305 2306bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2307 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2308 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2309} 2310 2311/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2312/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2313/// <0, 0, 1, 1> 2314bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2315 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2316 2317 unsigned NumElems = N->getNumOperands(); 2318 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2319 return false; 2320 2321 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2322 SDValue BitI = N->getOperand(i); 2323 SDValue BitI1 = N->getOperand(i+1); 2324 2325 if (!isUndefOrEqual(BitI, j)) 2326 return false; 2327 if (!isUndefOrEqual(BitI1, j)) 2328 return false; 2329 } 2330 2331 return true; 2332} 2333 2334/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2335/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2336/// <2, 2, 3, 3> 2337bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2338 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2339 2340 unsigned NumElems = N->getNumOperands(); 2341 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2342 return false; 2343 2344 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2345 SDValue BitI = N->getOperand(i); 2346 SDValue BitI1 = N->getOperand(i + 1); 2347 2348 if (!isUndefOrEqual(BitI, j)) 2349 return false; 2350 if (!isUndefOrEqual(BitI1, j)) 2351 return false; 2352 } 2353 2354 return true; 2355} 2356 2357/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2358/// specifies a shuffle of elements that is suitable for input to MOVSS, 2359/// MOVSD, and MOVD, i.e. setting the lowest element. 2360static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2361 if (NumElts != 2 && NumElts != 4) 2362 return false; 2363 2364 if (!isUndefOrEqual(Elts[0], NumElts)) 2365 return false; 2366 2367 for (unsigned i = 1; i < NumElts; ++i) { 2368 if (!isUndefOrEqual(Elts[i], i)) 2369 return false; 2370 } 2371 2372 return true; 2373} 2374 2375bool X86::isMOVLMask(SDNode *N) { 2376 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2377 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2378} 2379 2380/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2381/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2382/// element of vector 2 and the other elements to come from vector 1 in order. 2383static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2384 bool V2IsSplat = false, 2385 bool V2IsUndef = false) { 2386 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2387 return false; 2388 2389 if (!isUndefOrEqual(Ops[0], 0)) 2390 return false; 2391 2392 for (unsigned i = 1; i < NumOps; ++i) { 2393 SDValue Arg = Ops[i]; 2394 if (!(isUndefOrEqual(Arg, i+NumOps) || 2395 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2396 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2397 return false; 2398 } 2399 2400 return true; 2401} 2402 2403static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2404 bool V2IsUndef = false) { 2405 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2406 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2407 V2IsSplat, V2IsUndef); 2408} 2409 2410/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2411/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2412bool X86::isMOVSHDUPMask(SDNode *N) { 2413 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2414 2415 if (N->getNumOperands() != 4) 2416 return false; 2417 2418 // Expect 1, 1, 3, 3 2419 for (unsigned i = 0; i < 2; ++i) { 2420 SDValue Arg = N->getOperand(i); 2421 if (Arg.getOpcode() == ISD::UNDEF) continue; 2422 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2423 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2424 if (Val != 1) return false; 2425 } 2426 2427 bool HasHi = false; 2428 for (unsigned i = 2; i < 4; ++i) { 2429 SDValue Arg = N->getOperand(i); 2430 if (Arg.getOpcode() == ISD::UNDEF) continue; 2431 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2432 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2433 if (Val != 3) return false; 2434 HasHi = true; 2435 } 2436 2437 // Don't use movshdup if it can be done with a shufps. 2438 return HasHi; 2439} 2440 2441/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2442/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2443bool X86::isMOVSLDUPMask(SDNode *N) { 2444 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2445 2446 if (N->getNumOperands() != 4) 2447 return false; 2448 2449 // Expect 0, 0, 2, 2 2450 for (unsigned i = 0; i < 2; ++i) { 2451 SDValue Arg = N->getOperand(i); 2452 if (Arg.getOpcode() == ISD::UNDEF) continue; 2453 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2454 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2455 if (Val != 0) return false; 2456 } 2457 2458 bool HasHi = false; 2459 for (unsigned i = 2; i < 4; ++i) { 2460 SDValue Arg = N->getOperand(i); 2461 if (Arg.getOpcode() == ISD::UNDEF) continue; 2462 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2463 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2464 if (Val != 2) return false; 2465 HasHi = true; 2466 } 2467 2468 // Don't use movshdup if it can be done with a shufps. 2469 return HasHi; 2470} 2471 2472/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2473/// specifies a identity operation on the LHS or RHS. 2474static bool isIdentityMask(SDNode *N, bool RHS = false) { 2475 unsigned NumElems = N->getNumOperands(); 2476 for (unsigned i = 0; i < NumElems; ++i) 2477 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2478 return false; 2479 return true; 2480} 2481 2482/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2483/// a splat of a single element. 2484static bool isSplatMask(SDNode *N) { 2485 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2486 2487 // This is a splat operation if each element of the permute is the same, and 2488 // if the value doesn't reference the second vector. 2489 unsigned NumElems = N->getNumOperands(); 2490 SDValue ElementBase; 2491 unsigned i = 0; 2492 for (; i != NumElems; ++i) { 2493 SDValue Elt = N->getOperand(i); 2494 if (isa<ConstantSDNode>(Elt)) { 2495 ElementBase = Elt; 2496 break; 2497 } 2498 } 2499 2500 if (!ElementBase.getNode()) 2501 return false; 2502 2503 for (; i != NumElems; ++i) { 2504 SDValue Arg = N->getOperand(i); 2505 if (Arg.getOpcode() == ISD::UNDEF) continue; 2506 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2507 if (Arg != ElementBase) return false; 2508 } 2509 2510 // Make sure it is a splat of the first vector operand. 2511 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2512} 2513 2514/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2515/// a splat of a single element and it's a 2 or 4 element mask. 2516bool X86::isSplatMask(SDNode *N) { 2517 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2518 2519 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2520 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2521 return false; 2522 return ::isSplatMask(N); 2523} 2524 2525/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2526/// specifies a splat of zero element. 2527bool X86::isSplatLoMask(SDNode *N) { 2528 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2529 2530 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2531 if (!isUndefOrEqual(N->getOperand(i), 0)) 2532 return false; 2533 return true; 2534} 2535 2536/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2537/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2538/// instructions. 2539unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2540 unsigned NumOperands = N->getNumOperands(); 2541 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2542 unsigned Mask = 0; 2543 for (unsigned i = 0; i < NumOperands; ++i) { 2544 unsigned Val = 0; 2545 SDValue Arg = N->getOperand(NumOperands-i-1); 2546 if (Arg.getOpcode() != ISD::UNDEF) 2547 Val = cast<ConstantSDNode>(Arg)->getValue(); 2548 if (Val >= NumOperands) Val -= NumOperands; 2549 Mask |= Val; 2550 if (i != NumOperands - 1) 2551 Mask <<= Shift; 2552 } 2553 2554 return Mask; 2555} 2556 2557/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2558/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2559/// instructions. 2560unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2561 unsigned Mask = 0; 2562 // 8 nodes, but we only care about the last 4. 2563 for (unsigned i = 7; i >= 4; --i) { 2564 unsigned Val = 0; 2565 SDValue Arg = N->getOperand(i); 2566 if (Arg.getOpcode() != ISD::UNDEF) 2567 Val = cast<ConstantSDNode>(Arg)->getValue(); 2568 Mask |= (Val - 4); 2569 if (i != 4) 2570 Mask <<= 2; 2571 } 2572 2573 return Mask; 2574} 2575 2576/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2577/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2578/// instructions. 2579unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2580 unsigned Mask = 0; 2581 // 8 nodes, but we only care about the first 4. 2582 for (int i = 3; i >= 0; --i) { 2583 unsigned Val = 0; 2584 SDValue Arg = N->getOperand(i); 2585 if (Arg.getOpcode() != ISD::UNDEF) 2586 Val = cast<ConstantSDNode>(Arg)->getValue(); 2587 Mask |= Val; 2588 if (i != 0) 2589 Mask <<= 2; 2590 } 2591 2592 return Mask; 2593} 2594 2595/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2596/// specifies a 8 element shuffle that can be broken into a pair of 2597/// PSHUFHW and PSHUFLW. 2598static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2599 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2600 2601 if (N->getNumOperands() != 8) 2602 return false; 2603 2604 // Lower quadword shuffled. 2605 for (unsigned i = 0; i != 4; ++i) { 2606 SDValue Arg = N->getOperand(i); 2607 if (Arg.getOpcode() == ISD::UNDEF) continue; 2608 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2609 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2610 if (Val >= 4) 2611 return false; 2612 } 2613 2614 // Upper quadword shuffled. 2615 for (unsigned i = 4; i != 8; ++i) { 2616 SDValue Arg = N->getOperand(i); 2617 if (Arg.getOpcode() == ISD::UNDEF) continue; 2618 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2619 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2620 if (Val < 4 || Val > 7) 2621 return false; 2622 } 2623 2624 return true; 2625} 2626 2627/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2628/// values in ther permute mask. 2629static SDValue CommuteVectorShuffle(SDValue Op, SDValue &V1, 2630 SDValue &V2, SDValue &Mask, 2631 SelectionDAG &DAG) { 2632 MVT VT = Op.getValueType(); 2633 MVT MaskVT = Mask.getValueType(); 2634 MVT EltVT = MaskVT.getVectorElementType(); 2635 unsigned NumElems = Mask.getNumOperands(); 2636 SmallVector<SDValue, 8> MaskVec; 2637 2638 for (unsigned i = 0; i != NumElems; ++i) { 2639 SDValue Arg = Mask.getOperand(i); 2640 if (Arg.getOpcode() == ISD::UNDEF) { 2641 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2642 continue; 2643 } 2644 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2645 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2646 if (Val < NumElems) 2647 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2648 else 2649 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2650 } 2651 2652 std::swap(V1, V2); 2653 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2654 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2655} 2656 2657/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2658/// the two vector operands have swapped position. 2659static 2660SDValue CommuteVectorShuffleMask(SDValue Mask, SelectionDAG &DAG) { 2661 MVT MaskVT = Mask.getValueType(); 2662 MVT EltVT = MaskVT.getVectorElementType(); 2663 unsigned NumElems = Mask.getNumOperands(); 2664 SmallVector<SDValue, 8> MaskVec; 2665 for (unsigned i = 0; i != NumElems; ++i) { 2666 SDValue Arg = Mask.getOperand(i); 2667 if (Arg.getOpcode() == ISD::UNDEF) { 2668 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2669 continue; 2670 } 2671 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2672 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2673 if (Val < NumElems) 2674 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2675 else 2676 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2677 } 2678 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2679} 2680 2681 2682/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2683/// match movhlps. The lower half elements should come from upper half of 2684/// V1 (and in order), and the upper half elements should come from the upper 2685/// half of V2 (and in order). 2686static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2687 unsigned NumElems = Mask->getNumOperands(); 2688 if (NumElems != 4) 2689 return false; 2690 for (unsigned i = 0, e = 2; i != e; ++i) 2691 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2692 return false; 2693 for (unsigned i = 2; i != 4; ++i) 2694 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2695 return false; 2696 return true; 2697} 2698 2699/// isScalarLoadToVector - Returns true if the node is a scalar load that 2700/// is promoted to a vector. It also returns the LoadSDNode by reference if 2701/// required. 2702static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2703 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2704 N = N->getOperand(0).getNode(); 2705 if (ISD::isNON_EXTLoad(N)) { 2706 if (LD) 2707 *LD = cast<LoadSDNode>(N); 2708 return true; 2709 } 2710 } 2711 return false; 2712} 2713 2714/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2715/// match movlp{s|d}. The lower half elements should come from lower half of 2716/// V1 (and in order), and the upper half elements should come from the upper 2717/// half of V2 (and in order). And since V1 will become the source of the 2718/// MOVLP, it must be either a vector load or a scalar load to vector. 2719static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2720 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2721 return false; 2722 // Is V2 is a vector load, don't do this transformation. We will try to use 2723 // load folding shufps op. 2724 if (ISD::isNON_EXTLoad(V2)) 2725 return false; 2726 2727 unsigned NumElems = Mask->getNumOperands(); 2728 if (NumElems != 2 && NumElems != 4) 2729 return false; 2730 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2731 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2732 return false; 2733 for (unsigned i = NumElems/2; i != NumElems; ++i) 2734 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2735 return false; 2736 return true; 2737} 2738 2739/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2740/// all the same. 2741static bool isSplatVector(SDNode *N) { 2742 if (N->getOpcode() != ISD::BUILD_VECTOR) 2743 return false; 2744 2745 SDValue SplatValue = N->getOperand(0); 2746 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2747 if (N->getOperand(i) != SplatValue) 2748 return false; 2749 return true; 2750} 2751 2752/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2753/// to an undef. 2754static bool isUndefShuffle(SDNode *N) { 2755 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2756 return false; 2757 2758 SDValue V1 = N->getOperand(0); 2759 SDValue V2 = N->getOperand(1); 2760 SDValue Mask = N->getOperand(2); 2761 unsigned NumElems = Mask.getNumOperands(); 2762 for (unsigned i = 0; i != NumElems; ++i) { 2763 SDValue Arg = Mask.getOperand(i); 2764 if (Arg.getOpcode() != ISD::UNDEF) { 2765 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2766 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2767 return false; 2768 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2769 return false; 2770 } 2771 } 2772 return true; 2773} 2774 2775/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2776/// constant +0.0. 2777static inline bool isZeroNode(SDValue Elt) { 2778 return ((isa<ConstantSDNode>(Elt) && 2779 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2780 (isa<ConstantFPSDNode>(Elt) && 2781 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2782} 2783 2784/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2785/// to an zero vector. 2786static bool isZeroShuffle(SDNode *N) { 2787 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2788 return false; 2789 2790 SDValue V1 = N->getOperand(0); 2791 SDValue V2 = N->getOperand(1); 2792 SDValue Mask = N->getOperand(2); 2793 unsigned NumElems = Mask.getNumOperands(); 2794 for (unsigned i = 0; i != NumElems; ++i) { 2795 SDValue Arg = Mask.getOperand(i); 2796 if (Arg.getOpcode() == ISD::UNDEF) 2797 continue; 2798 2799 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2800 if (Idx < NumElems) { 2801 unsigned Opc = V1.getNode()->getOpcode(); 2802 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) 2803 continue; 2804 if (Opc != ISD::BUILD_VECTOR || 2805 !isZeroNode(V1.getNode()->getOperand(Idx))) 2806 return false; 2807 } else if (Idx >= NumElems) { 2808 unsigned Opc = V2.getNode()->getOpcode(); 2809 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) 2810 continue; 2811 if (Opc != ISD::BUILD_VECTOR || 2812 !isZeroNode(V2.getNode()->getOperand(Idx - NumElems))) 2813 return false; 2814 } 2815 } 2816 return true; 2817} 2818 2819/// getZeroVector - Returns a vector of specified type with all zero elements. 2820/// 2821static SDValue getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2822 assert(VT.isVector() && "Expected a vector type"); 2823 2824 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2825 // type. This ensures they get CSE'd. 2826 SDValue Vec; 2827 if (VT.getSizeInBits() == 64) { // MMX 2828 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2829 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2830 } else if (HasSSE2) { // SSE2 2831 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2832 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2833 } else { // SSE1 2834 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2835 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2836 } 2837 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2838} 2839 2840/// getOnesVector - Returns a vector of specified type with all bits set. 2841/// 2842static SDValue getOnesVector(MVT VT, SelectionDAG &DAG) { 2843 assert(VT.isVector() && "Expected a vector type"); 2844 2845 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2846 // type. This ensures they get CSE'd. 2847 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 2848 SDValue Vec; 2849 if (VT.getSizeInBits() == 64) // MMX 2850 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2851 else // SSE 2852 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2853 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2854} 2855 2856 2857/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2858/// that point to V2 points to its first element. 2859static SDValue NormalizeMask(SDValue Mask, SelectionDAG &DAG) { 2860 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2861 2862 bool Changed = false; 2863 SmallVector<SDValue, 8> MaskVec; 2864 unsigned NumElems = Mask.getNumOperands(); 2865 for (unsigned i = 0; i != NumElems; ++i) { 2866 SDValue Arg = Mask.getOperand(i); 2867 if (Arg.getOpcode() != ISD::UNDEF) { 2868 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2869 if (Val > NumElems) { 2870 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2871 Changed = true; 2872 } 2873 } 2874 MaskVec.push_back(Arg); 2875 } 2876 2877 if (Changed) 2878 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2879 &MaskVec[0], MaskVec.size()); 2880 return Mask; 2881} 2882 2883/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2884/// operation of specified width. 2885static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2886 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2887 MVT BaseVT = MaskVT.getVectorElementType(); 2888 2889 SmallVector<SDValue, 8> MaskVec; 2890 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2891 for (unsigned i = 1; i != NumElems; ++i) 2892 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2893 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2894} 2895 2896/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2897/// of specified width. 2898static SDValue getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2899 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2900 MVT BaseVT = MaskVT.getVectorElementType(); 2901 SmallVector<SDValue, 8> MaskVec; 2902 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2903 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2904 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2905 } 2906 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2907} 2908 2909/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2910/// of specified width. 2911static SDValue getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2912 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2913 MVT BaseVT = MaskVT.getVectorElementType(); 2914 unsigned Half = NumElems/2; 2915 SmallVector<SDValue, 8> MaskVec; 2916 for (unsigned i = 0; i != Half; ++i) { 2917 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2918 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2919 } 2920 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2921} 2922 2923/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2924/// element #0 of a vector with the specified index, leaving the rest of the 2925/// elements in place. 2926static SDValue getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2927 SelectionDAG &DAG) { 2928 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2929 MVT BaseVT = MaskVT.getVectorElementType(); 2930 SmallVector<SDValue, 8> MaskVec; 2931 // Element #0 of the result gets the elt we are replacing. 2932 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2933 for (unsigned i = 1; i != NumElems; ++i) 2934 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2935 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2936} 2937 2938/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2939static SDValue PromoteSplat(SDValue Op, SelectionDAG &DAG, bool HasSSE2) { 2940 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2941 MVT VT = Op.getValueType(); 2942 if (PVT == VT) 2943 return Op; 2944 SDValue V1 = Op.getOperand(0); 2945 SDValue Mask = Op.getOperand(2); 2946 unsigned NumElems = Mask.getNumOperands(); 2947 // Special handling of v4f32 -> v4i32. 2948 if (VT != MVT::v4f32) { 2949 Mask = getUnpacklMask(NumElems, DAG); 2950 while (NumElems > 4) { 2951 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2952 NumElems >>= 1; 2953 } 2954 Mask = getZeroVector(MVT::v4i32, true, DAG); 2955 } 2956 2957 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2958 SDValue Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2959 DAG.getNode(ISD::UNDEF, PVT), Mask); 2960 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2961} 2962 2963/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2964/// vector of zero or undef vector. This produces a shuffle where the low 2965/// element of V2 is swizzled into the zero/undef vector, landing at element 2966/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2967static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 2968 bool isZero, bool HasSSE2, 2969 SelectionDAG &DAG) { 2970 MVT VT = V2.getValueType(); 2971 SDValue V1 = isZero 2972 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2973 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2974 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2975 MVT EVT = MaskVT.getVectorElementType(); 2976 SmallVector<SDValue, 16> MaskVec; 2977 for (unsigned i = 0; i != NumElems; ++i) 2978 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2979 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2980 else 2981 MaskVec.push_back(DAG.getConstant(i, EVT)); 2982 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2983 &MaskVec[0], MaskVec.size()); 2984 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2985} 2986 2987/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2988/// a shuffle that is zero. 2989static 2990unsigned getNumOfConsecutiveZeros(SDValue Op, SDValue Mask, 2991 unsigned NumElems, bool Low, 2992 SelectionDAG &DAG) { 2993 unsigned NumZeros = 0; 2994 for (unsigned i = 0; i < NumElems; ++i) { 2995 unsigned Index = Low ? i : NumElems-i-1; 2996 SDValue Idx = Mask.getOperand(Index); 2997 if (Idx.getOpcode() == ISD::UNDEF) { 2998 ++NumZeros; 2999 continue; 3000 } 3001 SDValue Elt = DAG.getShuffleScalarElt(Op.getNode(), Index); 3002 if (Elt.getNode() && isZeroNode(Elt)) 3003 ++NumZeros; 3004 else 3005 break; 3006 } 3007 return NumZeros; 3008} 3009 3010/// isVectorShift - Returns true if the shuffle can be implemented as a 3011/// logical left or right shift of a vector. 3012static bool isVectorShift(SDValue Op, SDValue Mask, SelectionDAG &DAG, 3013 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 3014 unsigned NumElems = Mask.getNumOperands(); 3015 3016 isLeft = true; 3017 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 3018 if (!NumZeros) { 3019 isLeft = false; 3020 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 3021 if (!NumZeros) 3022 return false; 3023 } 3024 3025 bool SeenV1 = false; 3026 bool SeenV2 = false; 3027 for (unsigned i = NumZeros; i < NumElems; ++i) { 3028 unsigned Val = isLeft ? (i - NumZeros) : i; 3029 SDValue Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 3030 if (Idx.getOpcode() == ISD::UNDEF) 3031 continue; 3032 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 3033 if (Index < NumElems) 3034 SeenV1 = true; 3035 else { 3036 Index -= NumElems; 3037 SeenV2 = true; 3038 } 3039 if (Index != Val) 3040 return false; 3041 } 3042 if (SeenV1 && SeenV2) 3043 return false; 3044 3045 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 3046 ShAmt = NumZeros; 3047 return true; 3048} 3049 3050 3051/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 3052/// 3053static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 3054 unsigned NumNonZero, unsigned NumZero, 3055 SelectionDAG &DAG, TargetLowering &TLI) { 3056 if (NumNonZero > 8) 3057 return SDValue(); 3058 3059 SDValue V(0, 0); 3060 bool First = true; 3061 for (unsigned i = 0; i < 16; ++i) { 3062 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3063 if (ThisIsNonZero && First) { 3064 if (NumZero) 3065 V = getZeroVector(MVT::v8i16, true, DAG); 3066 else 3067 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3068 First = false; 3069 } 3070 3071 if ((i & 1) != 0) { 3072 SDValue ThisElt(0, 0), LastElt(0, 0); 3073 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3074 if (LastIsNonZero) { 3075 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3076 } 3077 if (ThisIsNonZero) { 3078 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3079 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3080 ThisElt, DAG.getConstant(8, MVT::i8)); 3081 if (LastIsNonZero) 3082 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3083 } else 3084 ThisElt = LastElt; 3085 3086 if (ThisElt.getNode()) 3087 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3088 DAG.getIntPtrConstant(i/2)); 3089 } 3090 } 3091 3092 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3093} 3094 3095/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3096/// 3097static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 3098 unsigned NumNonZero, unsigned NumZero, 3099 SelectionDAG &DAG, TargetLowering &TLI) { 3100 if (NumNonZero > 4) 3101 return SDValue(); 3102 3103 SDValue V(0, 0); 3104 bool First = true; 3105 for (unsigned i = 0; i < 8; ++i) { 3106 bool isNonZero = (NonZeros & (1 << i)) != 0; 3107 if (isNonZero) { 3108 if (First) { 3109 if (NumZero) 3110 V = getZeroVector(MVT::v8i16, true, DAG); 3111 else 3112 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3113 First = false; 3114 } 3115 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3116 DAG.getIntPtrConstant(i)); 3117 } 3118 } 3119 3120 return V; 3121} 3122 3123/// getVShift - Return a vector logical shift node. 3124/// 3125static SDValue getVShift(bool isLeft, MVT VT, SDValue SrcOp, 3126 unsigned NumBits, SelectionDAG &DAG, 3127 const TargetLowering &TLI) { 3128 bool isMMX = VT.getSizeInBits() == 64; 3129 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3130 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3131 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3132 return DAG.getNode(ISD::BIT_CONVERT, VT, 3133 DAG.getNode(Opc, ShVT, SrcOp, 3134 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3135} 3136 3137SDValue 3138X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) { 3139 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3140 if (ISD::isBuildVectorAllZeros(Op.getNode()) 3141 || ISD::isBuildVectorAllOnes(Op.getNode())) { 3142 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3143 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3144 // eliminated on x86-32 hosts. 3145 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3146 return Op; 3147 3148 if (ISD::isBuildVectorAllOnes(Op.getNode())) 3149 return getOnesVector(Op.getValueType(), DAG); 3150 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3151 } 3152 3153 MVT VT = Op.getValueType(); 3154 MVT EVT = VT.getVectorElementType(); 3155 unsigned EVTBits = EVT.getSizeInBits(); 3156 3157 unsigned NumElems = Op.getNumOperands(); 3158 unsigned NumZero = 0; 3159 unsigned NumNonZero = 0; 3160 unsigned NonZeros = 0; 3161 bool IsAllConstants = true; 3162 SmallSet<SDValue, 8> Values; 3163 for (unsigned i = 0; i < NumElems; ++i) { 3164 SDValue Elt = Op.getOperand(i); 3165 if (Elt.getOpcode() == ISD::UNDEF) 3166 continue; 3167 Values.insert(Elt); 3168 if (Elt.getOpcode() != ISD::Constant && 3169 Elt.getOpcode() != ISD::ConstantFP) 3170 IsAllConstants = false; 3171 if (isZeroNode(Elt)) 3172 NumZero++; 3173 else { 3174 NonZeros |= (1 << i); 3175 NumNonZero++; 3176 } 3177 } 3178 3179 if (NumNonZero == 0) { 3180 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3181 return DAG.getNode(ISD::UNDEF, VT); 3182 } 3183 3184 // Special case for single non-zero, non-undef, element. 3185 if (NumNonZero == 1 && NumElems <= 4) { 3186 unsigned Idx = CountTrailingZeros_32(NonZeros); 3187 SDValue Item = Op.getOperand(Idx); 3188 3189 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3190 // the value are obviously zero, truncate the value to i32 and do the 3191 // insertion that way. Only do this if the value is non-constant or if the 3192 // value is a constant being inserted into element 0. It is cheaper to do 3193 // a constant pool load than it is to do a movd + shuffle. 3194 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3195 (!IsAllConstants || Idx == 0)) { 3196 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3197 // Handle MMX and SSE both. 3198 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3199 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3200 3201 // Truncate the value (which may itself be a constant) to i32, and 3202 // convert it to a vector with movd (S2V+shuffle to zero extend). 3203 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3204 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3205 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3206 Subtarget->hasSSE2(), DAG); 3207 3208 // Now we have our 32-bit value zero extended in the low element of 3209 // a vector. If Idx != 0, swizzle it into place. 3210 if (Idx != 0) { 3211 SDValue Ops[] = { 3212 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3213 getSwapEltZeroMask(VecElts, Idx, DAG) 3214 }; 3215 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3216 } 3217 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3218 } 3219 } 3220 3221 // If we have a constant or non-constant insertion into the low element of 3222 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3223 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3224 // depending on what the source datatype is. Because we can only get here 3225 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3226 if (Idx == 0 && 3227 // Don't do this for i64 values on x86-32. 3228 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3229 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3230 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3231 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3232 Subtarget->hasSSE2(), DAG); 3233 } 3234 3235 // Is it a vector logical left shift? 3236 if (NumElems == 2 && Idx == 1 && 3237 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3238 unsigned NumBits = VT.getSizeInBits(); 3239 return getVShift(true, VT, 3240 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3241 NumBits/2, DAG, *this); 3242 } 3243 3244 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3245 return SDValue(); 3246 3247 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3248 // is a non-constant being inserted into an element other than the low one, 3249 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3250 // movd/movss) to move this into the low element, then shuffle it into 3251 // place. 3252 if (EVTBits == 32) { 3253 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3254 3255 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3256 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3257 Subtarget->hasSSE2(), DAG); 3258 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3259 MVT MaskEVT = MaskVT.getVectorElementType(); 3260 SmallVector<SDValue, 8> MaskVec; 3261 for (unsigned i = 0; i < NumElems; i++) 3262 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3263 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3264 &MaskVec[0], MaskVec.size()); 3265 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3266 DAG.getNode(ISD::UNDEF, VT), Mask); 3267 } 3268 } 3269 3270 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3271 if (Values.size() == 1) 3272 return SDValue(); 3273 3274 // A vector full of immediates; various special cases are already 3275 // handled, so this is best done with a single constant-pool load. 3276 if (IsAllConstants) 3277 return SDValue(); 3278 3279 // Let legalizer expand 2-wide build_vectors. 3280 if (EVTBits == 64) { 3281 if (NumNonZero == 1) { 3282 // One half is zero or undef. 3283 unsigned Idx = CountTrailingZeros_32(NonZeros); 3284 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3285 Op.getOperand(Idx)); 3286 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3287 Subtarget->hasSSE2(), DAG); 3288 } 3289 return SDValue(); 3290 } 3291 3292 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3293 if (EVTBits == 8 && NumElems == 16) { 3294 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3295 *this); 3296 if (V.getNode()) return V; 3297 } 3298 3299 if (EVTBits == 16 && NumElems == 8) { 3300 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3301 *this); 3302 if (V.getNode()) return V; 3303 } 3304 3305 // If element VT is == 32 bits, turn it into a number of shuffles. 3306 SmallVector<SDValue, 8> V; 3307 V.resize(NumElems); 3308 if (NumElems == 4 && NumZero > 0) { 3309 for (unsigned i = 0; i < 4; ++i) { 3310 bool isZero = !(NonZeros & (1 << i)); 3311 if (isZero) 3312 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3313 else 3314 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3315 } 3316 3317 for (unsigned i = 0; i < 2; ++i) { 3318 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3319 default: break; 3320 case 0: 3321 V[i] = V[i*2]; // Must be a zero vector. 3322 break; 3323 case 1: 3324 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3325 getMOVLMask(NumElems, DAG)); 3326 break; 3327 case 2: 3328 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3329 getMOVLMask(NumElems, DAG)); 3330 break; 3331 case 3: 3332 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3333 getUnpacklMask(NumElems, DAG)); 3334 break; 3335 } 3336 } 3337 3338 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3339 MVT EVT = MaskVT.getVectorElementType(); 3340 SmallVector<SDValue, 8> MaskVec; 3341 bool Reverse = (NonZeros & 0x3) == 2; 3342 for (unsigned i = 0; i < 2; ++i) 3343 if (Reverse) 3344 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3345 else 3346 MaskVec.push_back(DAG.getConstant(i, EVT)); 3347 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3348 for (unsigned i = 0; i < 2; ++i) 3349 if (Reverse) 3350 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3351 else 3352 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3353 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3354 &MaskVec[0], MaskVec.size()); 3355 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3356 } 3357 3358 if (Values.size() > 2) { 3359 // Expand into a number of unpckl*. 3360 // e.g. for v4f32 3361 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3362 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3363 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3364 SDValue UnpckMask = getUnpacklMask(NumElems, DAG); 3365 for (unsigned i = 0; i < NumElems; ++i) 3366 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3367 NumElems >>= 1; 3368 while (NumElems != 0) { 3369 for (unsigned i = 0; i < NumElems; ++i) 3370 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3371 UnpckMask); 3372 NumElems >>= 1; 3373 } 3374 return V[0]; 3375 } 3376 3377 return SDValue(); 3378} 3379 3380static 3381SDValue LowerVECTOR_SHUFFLEv8i16(SDValue V1, SDValue V2, 3382 SDValue PermMask, SelectionDAG &DAG, 3383 TargetLowering &TLI) { 3384 SDValue NewV; 3385 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3386 MVT MaskEVT = MaskVT.getVectorElementType(); 3387 MVT PtrVT = TLI.getPointerTy(); 3388 SmallVector<SDValue, 8> MaskElts(PermMask.getNode()->op_begin(), 3389 PermMask.getNode()->op_end()); 3390 3391 // First record which half of which vector the low elements come from. 3392 SmallVector<unsigned, 4> LowQuad(4); 3393 for (unsigned i = 0; i < 4; ++i) { 3394 SDValue Elt = MaskElts[i]; 3395 if (Elt.getOpcode() == ISD::UNDEF) 3396 continue; 3397 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3398 int QuadIdx = EltIdx / 4; 3399 ++LowQuad[QuadIdx]; 3400 } 3401 3402 int BestLowQuad = -1; 3403 unsigned MaxQuad = 1; 3404 for (unsigned i = 0; i < 4; ++i) { 3405 if (LowQuad[i] > MaxQuad) { 3406 BestLowQuad = i; 3407 MaxQuad = LowQuad[i]; 3408 } 3409 } 3410 3411 // Record which half of which vector the high elements come from. 3412 SmallVector<unsigned, 4> HighQuad(4); 3413 for (unsigned i = 4; i < 8; ++i) { 3414 SDValue Elt = MaskElts[i]; 3415 if (Elt.getOpcode() == ISD::UNDEF) 3416 continue; 3417 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3418 int QuadIdx = EltIdx / 4; 3419 ++HighQuad[QuadIdx]; 3420 } 3421 3422 int BestHighQuad = -1; 3423 MaxQuad = 1; 3424 for (unsigned i = 0; i < 4; ++i) { 3425 if (HighQuad[i] > MaxQuad) { 3426 BestHighQuad = i; 3427 MaxQuad = HighQuad[i]; 3428 } 3429 } 3430 3431 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3432 if (BestLowQuad != -1 || BestHighQuad != -1) { 3433 // First sort the 4 chunks in order using shufpd. 3434 SmallVector<SDValue, 8> MaskVec; 3435 3436 if (BestLowQuad != -1) 3437 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3438 else 3439 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3440 3441 if (BestHighQuad != -1) 3442 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3443 else 3444 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3445 3446 SDValue Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3447 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3448 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3449 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3450 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3451 3452 // Now sort high and low parts separately. 3453 BitVector InOrder(8); 3454 if (BestLowQuad != -1) { 3455 // Sort lower half in order using PSHUFLW. 3456 MaskVec.clear(); 3457 bool AnyOutOrder = false; 3458 3459 for (unsigned i = 0; i != 4; ++i) { 3460 SDValue Elt = MaskElts[i]; 3461 if (Elt.getOpcode() == ISD::UNDEF) { 3462 MaskVec.push_back(Elt); 3463 InOrder.set(i); 3464 } else { 3465 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3466 if (EltIdx != i) 3467 AnyOutOrder = true; 3468 3469 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3470 3471 // If this element is in the right place after this shuffle, then 3472 // remember it. 3473 if ((int)(EltIdx / 4) == BestLowQuad) 3474 InOrder.set(i); 3475 } 3476 } 3477 if (AnyOutOrder) { 3478 for (unsigned i = 4; i != 8; ++i) 3479 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3480 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3481 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3482 } 3483 } 3484 3485 if (BestHighQuad != -1) { 3486 // Sort high half in order using PSHUFHW if possible. 3487 MaskVec.clear(); 3488 3489 for (unsigned i = 0; i != 4; ++i) 3490 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3491 3492 bool AnyOutOrder = false; 3493 for (unsigned i = 4; i != 8; ++i) { 3494 SDValue Elt = MaskElts[i]; 3495 if (Elt.getOpcode() == ISD::UNDEF) { 3496 MaskVec.push_back(Elt); 3497 InOrder.set(i); 3498 } else { 3499 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3500 if (EltIdx != i) 3501 AnyOutOrder = true; 3502 3503 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3504 3505 // If this element is in the right place after this shuffle, then 3506 // remember it. 3507 if ((int)(EltIdx / 4) == BestHighQuad) 3508 InOrder.set(i); 3509 } 3510 } 3511 3512 if (AnyOutOrder) { 3513 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3514 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3515 } 3516 } 3517 3518 // The other elements are put in the right place using pextrw and pinsrw. 3519 for (unsigned i = 0; i != 8; ++i) { 3520 if (InOrder[i]) 3521 continue; 3522 SDValue Elt = MaskElts[i]; 3523 if (Elt.getOpcode() == ISD::UNDEF) 3524 continue; 3525 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3526 SDValue ExtOp = (EltIdx < 8) 3527 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3528 DAG.getConstant(EltIdx, PtrVT)) 3529 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3530 DAG.getConstant(EltIdx - 8, PtrVT)); 3531 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3532 DAG.getConstant(i, PtrVT)); 3533 } 3534 3535 return NewV; 3536 } 3537 3538 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use as 3539 // few as possible. First, let's find out how many elements are already in the 3540 // right order. 3541 unsigned V1InOrder = 0; 3542 unsigned V1FromV1 = 0; 3543 unsigned V2InOrder = 0; 3544 unsigned V2FromV2 = 0; 3545 SmallVector<SDValue, 8> V1Elts; 3546 SmallVector<SDValue, 8> V2Elts; 3547 for (unsigned i = 0; i < 8; ++i) { 3548 SDValue Elt = MaskElts[i]; 3549 if (Elt.getOpcode() == ISD::UNDEF) { 3550 V1Elts.push_back(Elt); 3551 V2Elts.push_back(Elt); 3552 ++V1InOrder; 3553 ++V2InOrder; 3554 continue; 3555 } 3556 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3557 if (EltIdx == i) { 3558 V1Elts.push_back(Elt); 3559 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3560 ++V1InOrder; 3561 } else if (EltIdx == i+8) { 3562 V1Elts.push_back(Elt); 3563 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3564 ++V2InOrder; 3565 } else if (EltIdx < 8) { 3566 V1Elts.push_back(Elt); 3567 ++V1FromV1; 3568 } else { 3569 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3570 ++V2FromV2; 3571 } 3572 } 3573 3574 if (V2InOrder > V1InOrder) { 3575 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3576 std::swap(V1, V2); 3577 std::swap(V1Elts, V2Elts); 3578 std::swap(V1FromV1, V2FromV2); 3579 } 3580 3581 if ((V1FromV1 + V1InOrder) != 8) { 3582 // Some elements are from V2. 3583 if (V1FromV1) { 3584 // If there are elements that are from V1 but out of place, 3585 // then first sort them in place 3586 SmallVector<SDValue, 8> MaskVec; 3587 for (unsigned i = 0; i < 8; ++i) { 3588 SDValue Elt = V1Elts[i]; 3589 if (Elt.getOpcode() == ISD::UNDEF) { 3590 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3591 continue; 3592 } 3593 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3594 if (EltIdx >= 8) 3595 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3596 else 3597 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3598 } 3599 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3600 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3601 } 3602 3603 NewV = V1; 3604 for (unsigned i = 0; i < 8; ++i) { 3605 SDValue Elt = V1Elts[i]; 3606 if (Elt.getOpcode() == ISD::UNDEF) 3607 continue; 3608 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3609 if (EltIdx < 8) 3610 continue; 3611 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3612 DAG.getConstant(EltIdx - 8, PtrVT)); 3613 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3614 DAG.getConstant(i, PtrVT)); 3615 } 3616 return NewV; 3617 } else { 3618 // All elements are from V1. 3619 NewV = V1; 3620 for (unsigned i = 0; i < 8; ++i) { 3621 SDValue Elt = V1Elts[i]; 3622 if (Elt.getOpcode() == ISD::UNDEF) 3623 continue; 3624 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3625 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3626 DAG.getConstant(EltIdx, PtrVT)); 3627 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3628 DAG.getConstant(i, PtrVT)); 3629 } 3630 return NewV; 3631 } 3632} 3633 3634/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3635/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3636/// done when every pair / quad of shuffle mask elements point to elements in 3637/// the right sequence. e.g. 3638/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3639static 3640SDValue RewriteAsNarrowerShuffle(SDValue V1, SDValue V2, 3641 MVT VT, 3642 SDValue PermMask, SelectionDAG &DAG, 3643 TargetLowering &TLI) { 3644 unsigned NumElems = PermMask.getNumOperands(); 3645 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3646 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3647 MVT MaskEltVT = MaskVT.getVectorElementType(); 3648 MVT NewVT = MaskVT; 3649 switch (VT.getSimpleVT()) { 3650 default: assert(false && "Unexpected!"); 3651 case MVT::v4f32: NewVT = MVT::v2f64; break; 3652 case MVT::v4i32: NewVT = MVT::v2i64; break; 3653 case MVT::v8i16: NewVT = MVT::v4i32; break; 3654 case MVT::v16i8: NewVT = MVT::v4i32; break; 3655 } 3656 3657 if (NewWidth == 2) { 3658 if (VT.isInteger()) 3659 NewVT = MVT::v2i64; 3660 else 3661 NewVT = MVT::v2f64; 3662 } 3663 unsigned Scale = NumElems / NewWidth; 3664 SmallVector<SDValue, 8> MaskVec; 3665 for (unsigned i = 0; i < NumElems; i += Scale) { 3666 unsigned StartIdx = ~0U; 3667 for (unsigned j = 0; j < Scale; ++j) { 3668 SDValue Elt = PermMask.getOperand(i+j); 3669 if (Elt.getOpcode() == ISD::UNDEF) 3670 continue; 3671 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3672 if (StartIdx == ~0U) 3673 StartIdx = EltIdx - (EltIdx % Scale); 3674 if (EltIdx != StartIdx + j) 3675 return SDValue(); 3676 } 3677 if (StartIdx == ~0U) 3678 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEltVT)); 3679 else 3680 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MaskEltVT)); 3681 } 3682 3683 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3684 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3685 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3686 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3687 &MaskVec[0], MaskVec.size())); 3688} 3689 3690/// getVZextMovL - Return a zero-extending vector move low node. 3691/// 3692static SDValue getVZextMovL(MVT VT, MVT OpVT, 3693 SDValue SrcOp, SelectionDAG &DAG, 3694 const X86Subtarget *Subtarget) { 3695 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3696 LoadSDNode *LD = NULL; 3697 if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) 3698 LD = dyn_cast<LoadSDNode>(SrcOp); 3699 if (!LD) { 3700 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3701 // instead. 3702 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3703 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3704 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3705 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3706 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3707 // PR2108 3708 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3709 return DAG.getNode(ISD::BIT_CONVERT, VT, 3710 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3711 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3712 SrcOp.getOperand(0) 3713 .getOperand(0)))); 3714 } 3715 } 3716 } 3717 3718 return DAG.getNode(ISD::BIT_CONVERT, VT, 3719 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3720 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3721} 3722 3723/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of 3724/// shuffles. 3725static SDValue 3726LowerVECTOR_SHUFFLE_4wide(SDValue V1, SDValue V2, 3727 SDValue PermMask, MVT VT, SelectionDAG &DAG) { 3728 MVT MaskVT = PermMask.getValueType(); 3729 MVT MaskEVT = MaskVT.getVectorElementType(); 3730 SmallVector<std::pair<int, int>, 8> Locs; 3731 Locs.resize(4); 3732 SmallVector<SDValue, 8> Mask1(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3733 unsigned NumHi = 0; 3734 unsigned NumLo = 0; 3735 for (unsigned i = 0; i != 4; ++i) { 3736 SDValue Elt = PermMask.getOperand(i); 3737 if (Elt.getOpcode() == ISD::UNDEF) { 3738 Locs[i] = std::make_pair(-1, -1); 3739 } else { 3740 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3741 assert(Val < 8 && "Invalid VECTOR_SHUFFLE index!"); 3742 if (Val < 4) { 3743 Locs[i] = std::make_pair(0, NumLo); 3744 Mask1[NumLo] = Elt; 3745 NumLo++; 3746 } else { 3747 Locs[i] = std::make_pair(1, NumHi); 3748 if (2+NumHi < 4) 3749 Mask1[2+NumHi] = Elt; 3750 NumHi++; 3751 } 3752 } 3753 } 3754 3755 if (NumLo <= 2 && NumHi <= 2) { 3756 // If no more than two elements come from either vector. This can be 3757 // implemented with two shuffles. First shuffle gather the elements. 3758 // The second shuffle, which takes the first shuffle as both of its 3759 // vector operands, put the elements into the right order. 3760 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3761 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3762 &Mask1[0], Mask1.size())); 3763 3764 SmallVector<SDValue, 8> Mask2(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3765 for (unsigned i = 0; i != 4; ++i) { 3766 if (Locs[i].first == -1) 3767 continue; 3768 else { 3769 unsigned Idx = (i < 2) ? 0 : 4; 3770 Idx += Locs[i].first * 2 + Locs[i].second; 3771 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3772 } 3773 } 3774 3775 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3776 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3777 &Mask2[0], Mask2.size())); 3778 } else if (NumLo == 3 || NumHi == 3) { 3779 // Otherwise, we must have three elements from one vector, call it X, and 3780 // one element from the other, call it Y. First, use a shufps to build an 3781 // intermediate vector with the one element from Y and the element from X 3782 // that will be in the same half in the final destination (the indexes don't 3783 // matter). Then, use a shufps to build the final vector, taking the half 3784 // containing the element from Y from the intermediate, and the other half 3785 // from X. 3786 if (NumHi == 3) { 3787 // Normalize it so the 3 elements come from V1. 3788 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3789 std::swap(V1, V2); 3790 } 3791 3792 // Find the element from V2. 3793 unsigned HiIndex; 3794 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 3795 SDValue Elt = PermMask.getOperand(HiIndex); 3796 if (Elt.getOpcode() == ISD::UNDEF) 3797 continue; 3798 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3799 if (Val >= 4) 3800 break; 3801 } 3802 3803 Mask1[0] = PermMask.getOperand(HiIndex); 3804 Mask1[1] = DAG.getNode(ISD::UNDEF, MaskEVT); 3805 Mask1[2] = PermMask.getOperand(HiIndex^1); 3806 Mask1[3] = DAG.getNode(ISD::UNDEF, MaskEVT); 3807 V2 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3808 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3809 3810 if (HiIndex >= 2) { 3811 Mask1[0] = PermMask.getOperand(0); 3812 Mask1[1] = PermMask.getOperand(1); 3813 Mask1[2] = DAG.getConstant(HiIndex & 1 ? 6 : 4, MaskEVT); 3814 Mask1[3] = DAG.getConstant(HiIndex & 1 ? 4 : 6, MaskEVT); 3815 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3816 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3817 } else { 3818 Mask1[0] = DAG.getConstant(HiIndex & 1 ? 2 : 0, MaskEVT); 3819 Mask1[1] = DAG.getConstant(HiIndex & 1 ? 0 : 2, MaskEVT); 3820 Mask1[2] = PermMask.getOperand(2); 3821 Mask1[3] = PermMask.getOperand(3); 3822 if (Mask1[2].getOpcode() != ISD::UNDEF) 3823 Mask1[2] = DAG.getConstant(cast<ConstantSDNode>(Mask1[2])->getValue()+4, 3824 MaskEVT); 3825 if (Mask1[3].getOpcode() != ISD::UNDEF) 3826 Mask1[3] = DAG.getConstant(cast<ConstantSDNode>(Mask1[3])->getValue()+4, 3827 MaskEVT); 3828 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, 3829 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3830 } 3831 } 3832 3833 // Break it into (shuffle shuffle_hi, shuffle_lo). 3834 Locs.clear(); 3835 SmallVector<SDValue,8> LoMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3836 SmallVector<SDValue,8> HiMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3837 SmallVector<SDValue,8> *MaskPtr = &LoMask; 3838 unsigned MaskIdx = 0; 3839 unsigned LoIdx = 0; 3840 unsigned HiIdx = 2; 3841 for (unsigned i = 0; i != 4; ++i) { 3842 if (i == 2) { 3843 MaskPtr = &HiMask; 3844 MaskIdx = 1; 3845 LoIdx = 0; 3846 HiIdx = 2; 3847 } 3848 SDValue Elt = PermMask.getOperand(i); 3849 if (Elt.getOpcode() == ISD::UNDEF) { 3850 Locs[i] = std::make_pair(-1, -1); 3851 } else if (cast<ConstantSDNode>(Elt)->getValue() < 4) { 3852 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3853 (*MaskPtr)[LoIdx] = Elt; 3854 LoIdx++; 3855 } else { 3856 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3857 (*MaskPtr)[HiIdx] = Elt; 3858 HiIdx++; 3859 } 3860 } 3861 3862 SDValue LoShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3863 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3864 &LoMask[0], LoMask.size())); 3865 SDValue HiShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3866 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3867 &HiMask[0], HiMask.size())); 3868 SmallVector<SDValue, 8> MaskOps; 3869 for (unsigned i = 0; i != 4; ++i) { 3870 if (Locs[i].first == -1) { 3871 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3872 } else { 3873 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 3874 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3875 } 3876 } 3877 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3878 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3879 &MaskOps[0], MaskOps.size())); 3880} 3881 3882SDValue 3883X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 3884 SDValue V1 = Op.getOperand(0); 3885 SDValue V2 = Op.getOperand(1); 3886 SDValue PermMask = Op.getOperand(2); 3887 MVT VT = Op.getValueType(); 3888 unsigned NumElems = PermMask.getNumOperands(); 3889 bool isMMX = VT.getSizeInBits() == 64; 3890 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3891 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3892 bool V1IsSplat = false; 3893 bool V2IsSplat = false; 3894 3895 if (isUndefShuffle(Op.getNode())) 3896 return DAG.getNode(ISD::UNDEF, VT); 3897 3898 if (isZeroShuffle(Op.getNode())) 3899 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3900 3901 if (isIdentityMask(PermMask.getNode())) 3902 return V1; 3903 else if (isIdentityMask(PermMask.getNode(), true)) 3904 return V2; 3905 3906 if (isSplatMask(PermMask.getNode())) { 3907 if (isMMX || NumElems < 4) return Op; 3908 // Promote it to a v4{if}32 splat. 3909 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3910 } 3911 3912 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3913 // do it! 3914 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3915 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3916 if (NewOp.getNode()) 3917 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3918 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3919 // FIXME: Figure out a cleaner way to do this. 3920 // Try to make use of movq to zero out the top part. 3921 if (ISD::isBuildVectorAllZeros(V2.getNode())) { 3922 SDValue NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3923 DAG, *this); 3924 if (NewOp.getNode()) { 3925 SDValue NewV1 = NewOp.getOperand(0); 3926 SDValue NewV2 = NewOp.getOperand(1); 3927 SDValue NewMask = NewOp.getOperand(2); 3928 if (isCommutedMOVL(NewMask.getNode(), true, false)) { 3929 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3930 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3931 } 3932 } 3933 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { 3934 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3935 DAG, *this); 3936 if (NewOp.getNode() && X86::isMOVLMask(NewOp.getOperand(2).getNode())) 3937 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3938 DAG, Subtarget); 3939 } 3940 } 3941 3942 // Check if this can be converted into a logical shift. 3943 bool isLeft = false; 3944 unsigned ShAmt = 0; 3945 SDValue ShVal; 3946 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3947 if (isShift && ShVal.hasOneUse()) { 3948 // If the shifted value has multiple uses, it may be cheaper to use 3949 // v_set0 + movlhps or movhlps, etc. 3950 MVT EVT = VT.getVectorElementType(); 3951 ShAmt *= EVT.getSizeInBits(); 3952 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3953 } 3954 3955 if (X86::isMOVLMask(PermMask.getNode())) { 3956 if (V1IsUndef) 3957 return V2; 3958 if (ISD::isBuildVectorAllZeros(V1.getNode())) 3959 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3960 if (!isMMX) 3961 return Op; 3962 } 3963 3964 if (!isMMX && (X86::isMOVSHDUPMask(PermMask.getNode()) || 3965 X86::isMOVSLDUPMask(PermMask.getNode()) || 3966 X86::isMOVHLPSMask(PermMask.getNode()) || 3967 X86::isMOVHPMask(PermMask.getNode()) || 3968 X86::isMOVLPMask(PermMask.getNode()))) 3969 return Op; 3970 3971 if (ShouldXformToMOVHLPS(PermMask.getNode()) || 3972 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), PermMask.getNode())) 3973 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3974 3975 if (isShift) { 3976 // No better options. Use a vshl / vsrl. 3977 MVT EVT = VT.getVectorElementType(); 3978 ShAmt *= EVT.getSizeInBits(); 3979 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3980 } 3981 3982 bool Commuted = false; 3983 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3984 // 1,1,1,1 -> v8i16 though. 3985 V1IsSplat = isSplatVector(V1.getNode()); 3986 V2IsSplat = isSplatVector(V2.getNode()); 3987 3988 // Canonicalize the splat or undef, if present, to be on the RHS. 3989 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3990 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3991 std::swap(V1IsSplat, V2IsSplat); 3992 std::swap(V1IsUndef, V2IsUndef); 3993 Commuted = true; 3994 } 3995 3996 // FIXME: Figure out a cleaner way to do this. 3997 if (isCommutedMOVL(PermMask.getNode(), V2IsSplat, V2IsUndef)) { 3998 if (V2IsUndef) return V1; 3999 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 4000 if (V2IsSplat) { 4001 // V2 is a splat, so the mask may be malformed. That is, it may point 4002 // to any V2 element. The instruction selectior won't like this. Get 4003 // a corrected mask and commute to form a proper MOVS{S|D}. 4004 SDValue NewMask = getMOVLMask(NumElems, DAG); 4005 if (NewMask.getNode() != PermMask.getNode()) 4006 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4007 } 4008 return Op; 4009 } 4010 4011 if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || 4012 X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || 4013 X86::isUNPCKLMask(PermMask.getNode()) || 4014 X86::isUNPCKHMask(PermMask.getNode())) 4015 return Op; 4016 4017 if (V2IsSplat) { 4018 // Normalize mask so all entries that point to V2 points to its first 4019 // element then try to match unpck{h|l} again. If match, return a 4020 // new vector_shuffle with the corrected mask. 4021 SDValue NewMask = NormalizeMask(PermMask, DAG); 4022 if (NewMask.getNode() != PermMask.getNode()) { 4023 if (X86::isUNPCKLMask(PermMask.getNode(), true)) { 4024 SDValue NewMask = getUnpacklMask(NumElems, DAG); 4025 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4026 } else if (X86::isUNPCKHMask(PermMask.getNode(), true)) { 4027 SDValue NewMask = getUnpackhMask(NumElems, DAG); 4028 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4029 } 4030 } 4031 } 4032 4033 // Normalize the node to match x86 shuffle ops if needed 4034 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.getNode())) 4035 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 4036 4037 if (Commuted) { 4038 // Commute is back and try unpck* again. 4039 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 4040 if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || 4041 X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || 4042 X86::isUNPCKLMask(PermMask.getNode()) || 4043 X86::isUNPCKHMask(PermMask.getNode())) 4044 return Op; 4045 } 4046 4047 // Try PSHUF* first, then SHUFP*. 4048 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 4049 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 4050 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.getNode())) { 4051 if (V2.getOpcode() != ISD::UNDEF) 4052 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 4053 DAG.getNode(ISD::UNDEF, VT), PermMask); 4054 return Op; 4055 } 4056 4057 if (!isMMX) { 4058 if (Subtarget->hasSSE2() && 4059 (X86::isPSHUFDMask(PermMask.getNode()) || 4060 X86::isPSHUFHWMask(PermMask.getNode()) || 4061 X86::isPSHUFLWMask(PermMask.getNode()))) { 4062 MVT RVT = VT; 4063 if (VT == MVT::v4f32) { 4064 RVT = MVT::v4i32; 4065 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 4066 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 4067 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4068 } else if (V2.getOpcode() != ISD::UNDEF) 4069 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 4070 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4071 if (RVT != VT) 4072 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 4073 return Op; 4074 } 4075 4076 // Binary or unary shufps. 4077 if (X86::isSHUFPMask(PermMask.getNode()) || 4078 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.getNode()))) 4079 return Op; 4080 } 4081 4082 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 4083 if (VT == MVT::v8i16) { 4084 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 4085 if (NewOp.getNode()) 4086 return NewOp; 4087 } 4088 4089 // Handle all 4 wide cases with a number of shuffles except for MMX. 4090 if (NumElems == 4 && !isMMX) 4091 return LowerVECTOR_SHUFFLE_4wide(V1, V2, PermMask, VT, DAG); 4092 4093 return SDValue(); 4094} 4095 4096SDValue 4097X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 4098 SelectionDAG &DAG) { 4099 MVT VT = Op.getValueType(); 4100 if (VT.getSizeInBits() == 8) { 4101 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 4102 Op.getOperand(0), Op.getOperand(1)); 4103 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4104 DAG.getValueType(VT)); 4105 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4106 } else if (VT.getSizeInBits() == 16) { 4107 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 4108 Op.getOperand(0), Op.getOperand(1)); 4109 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4110 DAG.getValueType(VT)); 4111 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4112 } else if (VT == MVT::f32) { 4113 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 4114 // the result back to FR32 register. It's only worth matching if the 4115 // result has a single use which is a store or a bitcast to i32. 4116 if (!Op.hasOneUse()) 4117 return SDValue(); 4118 SDNode *User = *Op.getNode()->use_begin(); 4119 if (User->getOpcode() != ISD::STORE && 4120 (User->getOpcode() != ISD::BIT_CONVERT || 4121 User->getValueType(0) != MVT::i32)) 4122 return SDValue(); 4123 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4124 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 4125 Op.getOperand(1)); 4126 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 4127 } 4128 return SDValue(); 4129} 4130 4131 4132SDValue 4133X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4134 if (!isa<ConstantSDNode>(Op.getOperand(1))) 4135 return SDValue(); 4136 4137 if (Subtarget->hasSSE41()) { 4138 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 4139 if (Res.getNode()) 4140 return Res; 4141 } 4142 4143 MVT VT = Op.getValueType(); 4144 // TODO: handle v16i8. 4145 if (VT.getSizeInBits() == 16) { 4146 SDValue Vec = Op.getOperand(0); 4147 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4148 if (Idx == 0) 4149 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4150 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4151 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4152 Op.getOperand(1))); 4153 // Transform it so it match pextrw which produces a 32-bit result. 4154 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4155 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4156 Op.getOperand(0), Op.getOperand(1)); 4157 SDValue Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4158 DAG.getValueType(VT)); 4159 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4160 } else if (VT.getSizeInBits() == 32) { 4161 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4162 if (Idx == 0) 4163 return Op; 4164 // SHUFPS the element to the lowest double word, then movss. 4165 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4166 SmallVector<SDValue, 8> IdxVec; 4167 IdxVec. 4168 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4169 IdxVec. 4170 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4171 IdxVec. 4172 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4173 IdxVec. 4174 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4175 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4176 &IdxVec[0], IdxVec.size()); 4177 SDValue Vec = Op.getOperand(0); 4178 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4179 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4180 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4181 DAG.getIntPtrConstant(0)); 4182 } else if (VT.getSizeInBits() == 64) { 4183 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4184 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4185 // to match extract_elt for f64. 4186 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4187 if (Idx == 0) 4188 return Op; 4189 4190 // UNPCKHPD the element to the lowest double word, then movsd. 4191 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4192 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4193 MVT MaskVT = MVT::getIntVectorWithNumElements(2); 4194 SmallVector<SDValue, 8> IdxVec; 4195 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4196 IdxVec. 4197 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4198 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4199 &IdxVec[0], IdxVec.size()); 4200 SDValue Vec = Op.getOperand(0); 4201 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4202 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4203 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4204 DAG.getIntPtrConstant(0)); 4205 } 4206 4207 return SDValue(); 4208} 4209 4210SDValue 4211X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){ 4212 MVT VT = Op.getValueType(); 4213 MVT EVT = VT.getVectorElementType(); 4214 4215 SDValue N0 = Op.getOperand(0); 4216 SDValue N1 = Op.getOperand(1); 4217 SDValue N2 = Op.getOperand(2); 4218 4219 if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) && 4220 isa<ConstantSDNode>(N2)) { 4221 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4222 : X86ISD::PINSRW; 4223 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4224 // argument. 4225 if (N1.getValueType() != MVT::i32) 4226 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4227 if (N2.getValueType() != MVT::i32) 4228 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4229 return DAG.getNode(Opc, VT, N0, N1, N2); 4230 } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 4231 // Bits [7:6] of the constant are the source select. This will always be 4232 // zero here. The DAG Combiner may combine an extract_elt index into these 4233 // bits. For example (insert (extract, 3), 2) could be matched by putting 4234 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4235 // Bits [5:4] of the constant are the destination select. This is the 4236 // value of the incoming immediate. 4237 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4238 // combine either bitwise AND or insert of float 0.0 to set these bits. 4239 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4240 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4241 } 4242 return SDValue(); 4243} 4244 4245SDValue 4246X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4247 MVT VT = Op.getValueType(); 4248 MVT EVT = VT.getVectorElementType(); 4249 4250 if (Subtarget->hasSSE41()) 4251 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4252 4253 if (EVT == MVT::i8) 4254 return SDValue(); 4255 4256 SDValue N0 = Op.getOperand(0); 4257 SDValue N1 = Op.getOperand(1); 4258 SDValue N2 = Op.getOperand(2); 4259 4260 if (EVT.getSizeInBits() == 16) { 4261 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4262 // as its second argument. 4263 if (N1.getValueType() != MVT::i32) 4264 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4265 if (N2.getValueType() != MVT::i32) 4266 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4267 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4268 } 4269 return SDValue(); 4270} 4271 4272SDValue 4273X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) { 4274 if (Op.getValueType() == MVT::v2f32) 4275 return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f32, 4276 DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i32, 4277 DAG.getNode(ISD::BIT_CONVERT, MVT::i32, 4278 Op.getOperand(0)))); 4279 4280 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4281 MVT VT = MVT::v2i32; 4282 switch (Op.getValueType().getSimpleVT()) { 4283 default: break; 4284 case MVT::v16i8: 4285 case MVT::v8i16: 4286 VT = MVT::v4i32; 4287 break; 4288 } 4289 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4290 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4291} 4292 4293// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4294// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4295// one of the above mentioned nodes. It has to be wrapped because otherwise 4296// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4297// be used to form addressing mode. These wrapped nodes will be selected 4298// into MOV32ri. 4299SDValue 4300X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) { 4301 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4302 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), 4303 getPointerTy(), 4304 CP->getAlignment()); 4305 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4306 // With PIC, the address is actually $g + Offset. 4307 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4308 !Subtarget->isPICStyleRIPRel()) { 4309 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4310 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4311 Result); 4312 } 4313 4314 return Result; 4315} 4316 4317SDValue 4318X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) { 4319 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4320 SDValue Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4321 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4322 // With PIC, the address is actually $g + Offset. 4323 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4324 !Subtarget->isPICStyleRIPRel()) { 4325 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4326 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4327 Result); 4328 } 4329 4330 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4331 // load the value at address GV, not the value of GV itself. This means that 4332 // the GlobalAddress must be in the base or index register of the address, not 4333 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4334 // The same applies for external symbols during PIC codegen 4335 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4336 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4337 PseudoSourceValue::getGOT(), 0); 4338 4339 return Result; 4340} 4341 4342// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4343static SDValue 4344LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4345 const MVT PtrVT) { 4346 SDValue InFlag; 4347 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4348 DAG.getNode(X86ISD::GlobalBaseReg, 4349 PtrVT), InFlag); 4350 InFlag = Chain.getValue(1); 4351 4352 // emit leal symbol@TLSGD(,%ebx,1), %eax 4353 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4354 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4355 GA->getValueType(0), 4356 GA->getOffset()); 4357 SDValue Ops[] = { Chain, TGA, InFlag }; 4358 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4359 InFlag = Result.getValue(2); 4360 Chain = Result.getValue(1); 4361 4362 // call ___tls_get_addr. This function receives its argument in 4363 // the register EAX. 4364 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4365 InFlag = Chain.getValue(1); 4366 4367 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4368 SDValue Ops1[] = { Chain, 4369 DAG.getTargetExternalSymbol("___tls_get_addr", 4370 PtrVT), 4371 DAG.getRegister(X86::EAX, PtrVT), 4372 DAG.getRegister(X86::EBX, PtrVT), 4373 InFlag }; 4374 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4375 InFlag = Chain.getValue(1); 4376 4377 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4378} 4379 4380// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4381static SDValue 4382LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4383 const MVT PtrVT) { 4384 SDValue InFlag, Chain; 4385 4386 // emit leaq symbol@TLSGD(%rip), %rdi 4387 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4388 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4389 GA->getValueType(0), 4390 GA->getOffset()); 4391 SDValue Ops[] = { DAG.getEntryNode(), TGA}; 4392 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4393 Chain = Result.getValue(1); 4394 InFlag = Result.getValue(2); 4395 4396 // call __tls_get_addr. This function receives its argument in 4397 // the register RDI. 4398 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4399 InFlag = Chain.getValue(1); 4400 4401 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4402 SDValue Ops1[] = { Chain, 4403 DAG.getTargetExternalSymbol("__tls_get_addr", 4404 PtrVT), 4405 DAG.getRegister(X86::RDI, PtrVT), 4406 InFlag }; 4407 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4408 InFlag = Chain.getValue(1); 4409 4410 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4411} 4412 4413// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4414// "local exec" model. 4415static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4416 const MVT PtrVT) { 4417 // Get the Thread Pointer 4418 SDValue ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4419 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4420 // exec) 4421 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4422 GA->getValueType(0), 4423 GA->getOffset()); 4424 SDValue Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4425 4426 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4427 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4428 PseudoSourceValue::getGOT(), 0); 4429 4430 // The address of the thread local variable is the add of the thread 4431 // pointer with the offset of the variable. 4432 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4433} 4434 4435SDValue 4436X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) { 4437 // TODO: implement the "local dynamic" model 4438 // TODO: implement the "initial exec"model for pic executables 4439 assert(Subtarget->isTargetELF() && 4440 "TLS not implemented for non-ELF targets"); 4441 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4442 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4443 // otherwise use the "Local Exec"TLS Model 4444 if (Subtarget->is64Bit()) { 4445 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4446 } else { 4447 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4448 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4449 else 4450 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4451 } 4452} 4453 4454SDValue 4455X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) { 4456 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4457 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4458 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4459 // With PIC, the address is actually $g + Offset. 4460 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4461 !Subtarget->isPICStyleRIPRel()) { 4462 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4463 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4464 Result); 4465 } 4466 4467 return Result; 4468} 4469 4470SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) { 4471 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4472 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4473 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4474 // With PIC, the address is actually $g + Offset. 4475 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4476 !Subtarget->isPICStyleRIPRel()) { 4477 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4478 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4479 Result); 4480 } 4481 4482 return Result; 4483} 4484 4485/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4486/// take a 2 x i32 value to shift plus a shift amount. 4487SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) { 4488 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4489 MVT VT = Op.getValueType(); 4490 unsigned VTBits = VT.getSizeInBits(); 4491 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4492 SDValue ShOpLo = Op.getOperand(0); 4493 SDValue ShOpHi = Op.getOperand(1); 4494 SDValue ShAmt = Op.getOperand(2); 4495 SDValue Tmp1 = isSRA ? 4496 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4497 DAG.getConstant(0, VT); 4498 4499 SDValue Tmp2, Tmp3; 4500 if (Op.getOpcode() == ISD::SHL_PARTS) { 4501 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4502 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4503 } else { 4504 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4505 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4506 } 4507 4508 SDValue AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4509 DAG.getConstant(VTBits, MVT::i8)); 4510 SDValue Cond = DAG.getNode(X86ISD::CMP, VT, 4511 AndNode, DAG.getConstant(0, MVT::i8)); 4512 4513 SDValue Hi, Lo; 4514 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4515 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 4516 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 4517 4518 if (Op.getOpcode() == ISD::SHL_PARTS) { 4519 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4520 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4521 } else { 4522 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4523 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4524 } 4525 4526 SDValue Ops[2] = { Lo, Hi }; 4527 return DAG.getMergeValues(Ops, 2); 4528} 4529 4530SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 4531 MVT SrcVT = Op.getOperand(0).getValueType(); 4532 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4533 "Unknown SINT_TO_FP to lower!"); 4534 4535 // These are really Legal; caller falls through into that case. 4536 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4537 return SDValue(); 4538 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4539 Subtarget->is64Bit()) 4540 return SDValue(); 4541 4542 unsigned Size = SrcVT.getSizeInBits()/8; 4543 MachineFunction &MF = DAG.getMachineFunction(); 4544 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4545 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4546 SDValue Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4547 StackSlot, 4548 PseudoSourceValue::getFixedStack(SSFI), 0); 4549 4550 // Build the FILD 4551 SDVTList Tys; 4552 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4553 if (useSSE) 4554 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4555 else 4556 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4557 SmallVector<SDValue, 8> Ops; 4558 Ops.push_back(Chain); 4559 Ops.push_back(StackSlot); 4560 Ops.push_back(DAG.getValueType(SrcVT)); 4561 SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4562 Tys, &Ops[0], Ops.size()); 4563 4564 if (useSSE) { 4565 Chain = Result.getValue(1); 4566 SDValue InFlag = Result.getValue(2); 4567 4568 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4569 // shouldn't be necessary except that RFP cannot be live across 4570 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4571 MachineFunction &MF = DAG.getMachineFunction(); 4572 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4573 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4574 Tys = DAG.getVTList(MVT::Other); 4575 SmallVector<SDValue, 8> Ops; 4576 Ops.push_back(Chain); 4577 Ops.push_back(Result); 4578 Ops.push_back(StackSlot); 4579 Ops.push_back(DAG.getValueType(Op.getValueType())); 4580 Ops.push_back(InFlag); 4581 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4582 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4583 PseudoSourceValue::getFixedStack(SSFI), 0); 4584 } 4585 4586 return Result; 4587} 4588 4589std::pair<SDValue,SDValue> X86TargetLowering:: 4590FP_TO_SINTHelper(SDValue Op, SelectionDAG &DAG) { 4591 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4592 Op.getValueType().getSimpleVT() >= MVT::i16 && 4593 "Unknown FP_TO_SINT to lower!"); 4594 4595 // These are really Legal. 4596 if (Op.getValueType() == MVT::i32 && 4597 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4598 return std::make_pair(SDValue(), SDValue()); 4599 if (Subtarget->is64Bit() && 4600 Op.getValueType() == MVT::i64 && 4601 Op.getOperand(0).getValueType() != MVT::f80) 4602 return std::make_pair(SDValue(), SDValue()); 4603 4604 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4605 // stack slot. 4606 MachineFunction &MF = DAG.getMachineFunction(); 4607 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4608 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4609 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4610 unsigned Opc; 4611 switch (Op.getValueType().getSimpleVT()) { 4612 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4613 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4614 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4615 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4616 } 4617 4618 SDValue Chain = DAG.getEntryNode(); 4619 SDValue Value = Op.getOperand(0); 4620 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4621 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4622 Chain = DAG.getStore(Chain, Value, StackSlot, 4623 PseudoSourceValue::getFixedStack(SSFI), 0); 4624 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4625 SDValue Ops[] = { 4626 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4627 }; 4628 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4629 Chain = Value.getValue(1); 4630 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4631 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4632 } 4633 4634 // Build the FP_TO_INT*_IN_MEM 4635 SDValue Ops[] = { Chain, Value, StackSlot }; 4636 SDValue FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4637 4638 return std::make_pair(FIST, StackSlot); 4639} 4640 4641SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) { 4642 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(Op, DAG); 4643 SDValue FIST = Vals.first, StackSlot = Vals.second; 4644 if (FIST.getNode() == 0) return SDValue(); 4645 4646 // Load the result. 4647 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4648} 4649 4650SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4651 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG); 4652 SDValue FIST = Vals.first, StackSlot = Vals.second; 4653 if (FIST.getNode() == 0) return 0; 4654 4655 MVT VT = N->getValueType(0); 4656 4657 // Return a load from the stack slot. 4658 SDValue Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0); 4659 4660 // Use MERGE_VALUES to drop the chain result value and get a node with one 4661 // result. This requires turning off getMergeValues simplification, since 4662 // otherwise it will give us Res back. 4663 return DAG.getMergeValues(&Res, 1, false).getNode(); 4664} 4665 4666SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) { 4667 MVT VT = Op.getValueType(); 4668 MVT EltVT = VT; 4669 if (VT.isVector()) 4670 EltVT = VT.getVectorElementType(); 4671 std::vector<Constant*> CV; 4672 if (EltVT == MVT::f64) { 4673 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4674 CV.push_back(C); 4675 CV.push_back(C); 4676 } else { 4677 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4678 CV.push_back(C); 4679 CV.push_back(C); 4680 CV.push_back(C); 4681 CV.push_back(C); 4682 } 4683 Constant *C = ConstantVector::get(CV); 4684 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4685 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4686 PseudoSourceValue::getConstantPool(), 0, 4687 false, 16); 4688 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4689} 4690 4691SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) { 4692 MVT VT = Op.getValueType(); 4693 MVT EltVT = VT; 4694 unsigned EltNum = 1; 4695 if (VT.isVector()) { 4696 EltVT = VT.getVectorElementType(); 4697 EltNum = VT.getVectorNumElements(); 4698 } 4699 std::vector<Constant*> CV; 4700 if (EltVT == MVT::f64) { 4701 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4702 CV.push_back(C); 4703 CV.push_back(C); 4704 } else { 4705 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4706 CV.push_back(C); 4707 CV.push_back(C); 4708 CV.push_back(C); 4709 CV.push_back(C); 4710 } 4711 Constant *C = ConstantVector::get(CV); 4712 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4713 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4714 PseudoSourceValue::getConstantPool(), 0, 4715 false, 16); 4716 if (VT.isVector()) { 4717 return DAG.getNode(ISD::BIT_CONVERT, VT, 4718 DAG.getNode(ISD::XOR, MVT::v2i64, 4719 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4720 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4721 } else { 4722 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4723 } 4724} 4725 4726SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { 4727 SDValue Op0 = Op.getOperand(0); 4728 SDValue Op1 = Op.getOperand(1); 4729 MVT VT = Op.getValueType(); 4730 MVT SrcVT = Op1.getValueType(); 4731 4732 // If second operand is smaller, extend it first. 4733 if (SrcVT.bitsLT(VT)) { 4734 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4735 SrcVT = VT; 4736 } 4737 // And if it is bigger, shrink it first. 4738 if (SrcVT.bitsGT(VT)) { 4739 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4740 SrcVT = VT; 4741 } 4742 4743 // At this point the operands and the result should have the same 4744 // type, and that won't be f80 since that is not custom lowered. 4745 4746 // First get the sign bit of second operand. 4747 std::vector<Constant*> CV; 4748 if (SrcVT == MVT::f64) { 4749 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4750 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4751 } else { 4752 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4753 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4754 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4755 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4756 } 4757 Constant *C = ConstantVector::get(CV); 4758 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4759 SDValue Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4760 PseudoSourceValue::getConstantPool(), 0, 4761 false, 16); 4762 SDValue SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4763 4764 // Shift sign bit right or left if the two operands have different types. 4765 if (SrcVT.bitsGT(VT)) { 4766 // Op0 is MVT::f32, Op1 is MVT::f64. 4767 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4768 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4769 DAG.getConstant(32, MVT::i32)); 4770 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4771 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4772 DAG.getIntPtrConstant(0)); 4773 } 4774 4775 // Clear first operand sign bit. 4776 CV.clear(); 4777 if (VT == MVT::f64) { 4778 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4779 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4780 } else { 4781 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4782 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4783 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4784 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4785 } 4786 C = ConstantVector::get(CV); 4787 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4788 SDValue Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4789 PseudoSourceValue::getConstantPool(), 0, 4790 false, 16); 4791 SDValue Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4792 4793 // Or the value with the sign bit. 4794 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4795} 4796 4797SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) { 4798 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4799 SDValue Cond; 4800 SDValue Op0 = Op.getOperand(0); 4801 SDValue Op1 = Op.getOperand(1); 4802 SDValue CC = Op.getOperand(2); 4803 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4804 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4805 unsigned X86CC; 4806 4807 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4808 Op0, Op1, DAG)) { 4809 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4810 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4811 DAG.getConstant(X86CC, MVT::i8), Cond); 4812 } 4813 4814 assert(isFP && "Illegal integer SetCC!"); 4815 4816 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4817 switch (SetCCOpcode) { 4818 default: assert(false && "Illegal floating point SetCC!"); 4819 case ISD::SETOEQ: { // !PF & ZF 4820 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4821 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4822 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4823 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4824 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4825 } 4826 case ISD::SETUNE: { // PF | !ZF 4827 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4828 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4829 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4830 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4831 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4832 } 4833 } 4834} 4835 4836SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 4837 SDValue Cond; 4838 SDValue Op0 = Op.getOperand(0); 4839 SDValue Op1 = Op.getOperand(1); 4840 SDValue CC = Op.getOperand(2); 4841 MVT VT = Op.getValueType(); 4842 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4843 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4844 4845 if (isFP) { 4846 unsigned SSECC = 8; 4847 MVT VT0 = Op0.getValueType(); 4848 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); 4849 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; 4850 bool Swap = false; 4851 4852 switch (SetCCOpcode) { 4853 default: break; 4854 case ISD::SETOEQ: 4855 case ISD::SETEQ: SSECC = 0; break; 4856 case ISD::SETOGT: 4857 case ISD::SETGT: Swap = true; // Fallthrough 4858 case ISD::SETLT: 4859 case ISD::SETOLT: SSECC = 1; break; 4860 case ISD::SETOGE: 4861 case ISD::SETGE: Swap = true; // Fallthrough 4862 case ISD::SETLE: 4863 case ISD::SETOLE: SSECC = 2; break; 4864 case ISD::SETUO: SSECC = 3; break; 4865 case ISD::SETUNE: 4866 case ISD::SETNE: SSECC = 4; break; 4867 case ISD::SETULE: Swap = true; 4868 case ISD::SETUGE: SSECC = 5; break; 4869 case ISD::SETULT: Swap = true; 4870 case ISD::SETUGT: SSECC = 6; break; 4871 case ISD::SETO: SSECC = 7; break; 4872 } 4873 if (Swap) 4874 std::swap(Op0, Op1); 4875 4876 // In the two special cases we can't handle, emit two comparisons. 4877 if (SSECC == 8) { 4878 if (SetCCOpcode == ISD::SETUEQ) { 4879 SDValue UNORD, EQ; 4880 UNORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 4881 EQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 4882 return DAG.getNode(ISD::OR, VT, UNORD, EQ); 4883 } 4884 else if (SetCCOpcode == ISD::SETONE) { 4885 SDValue ORD, NEQ; 4886 ORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 4887 NEQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 4888 return DAG.getNode(ISD::AND, VT, ORD, NEQ); 4889 } 4890 assert(0 && "Illegal FP comparison"); 4891 } 4892 // Handle all other FP comparisons here. 4893 return DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 4894 } 4895 4896 // We are handling one of the integer comparisons here. Since SSE only has 4897 // GT and EQ comparisons for integer, swapping operands and multiple 4898 // operations may be required for some comparisons. 4899 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 4900 bool Swap = false, Invert = false, FlipSigns = false; 4901 4902 switch (VT.getSimpleVT()) { 4903 default: break; 4904 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 4905 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 4906 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 4907 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 4908 } 4909 4910 switch (SetCCOpcode) { 4911 default: break; 4912 case ISD::SETNE: Invert = true; 4913 case ISD::SETEQ: Opc = EQOpc; break; 4914 case ISD::SETLT: Swap = true; 4915 case ISD::SETGT: Opc = GTOpc; break; 4916 case ISD::SETGE: Swap = true; 4917 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 4918 case ISD::SETULT: Swap = true; 4919 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 4920 case ISD::SETUGE: Swap = true; 4921 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 4922 } 4923 if (Swap) 4924 std::swap(Op0, Op1); 4925 4926 // Since SSE has no unsigned integer comparisons, we need to flip the sign 4927 // bits of the inputs before performing those operations. 4928 if (FlipSigns) { 4929 MVT EltVT = VT.getVectorElementType(); 4930 SDValue SignBit = DAG.getConstant(EltVT.getIntegerVTSignBit(), EltVT); 4931 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 4932 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, VT, &SignBits[0], 4933 SignBits.size()); 4934 Op0 = DAG.getNode(ISD::XOR, VT, Op0, SignVec); 4935 Op1 = DAG.getNode(ISD::XOR, VT, Op1, SignVec); 4936 } 4937 4938 SDValue Result = DAG.getNode(Opc, VT, Op0, Op1); 4939 4940 // If the logical-not of the result is required, perform that now. 4941 if (Invert) { 4942 MVT EltVT = VT.getVectorElementType(); 4943 SDValue NegOne = DAG.getConstant(EltVT.getIntegerVTBitMask(), EltVT); 4944 std::vector<SDValue> NegOnes(VT.getVectorNumElements(), NegOne); 4945 SDValue NegOneV = DAG.getNode(ISD::BUILD_VECTOR, VT, &NegOnes[0], 4946 NegOnes.size()); 4947 Result = DAG.getNode(ISD::XOR, VT, Result, NegOneV); 4948 } 4949 return Result; 4950} 4951 4952SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) { 4953 bool addTest = true; 4954 SDValue Cond = Op.getOperand(0); 4955 SDValue CC; 4956 4957 if (Cond.getOpcode() == ISD::SETCC) 4958 Cond = LowerSETCC(Cond, DAG); 4959 4960 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4961 // setting operand in place of the X86ISD::SETCC. 4962 if (Cond.getOpcode() == X86ISD::SETCC) { 4963 CC = Cond.getOperand(0); 4964 4965 SDValue Cmp = Cond.getOperand(1); 4966 unsigned Opc = Cmp.getOpcode(); 4967 MVT VT = Op.getValueType(); 4968 4969 bool IllegalFPCMov = false; 4970 if (VT.isFloatingPoint() && !VT.isVector() && 4971 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4972 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4973 4974 if ((Opc == X86ISD::CMP || 4975 Opc == X86ISD::COMI || 4976 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4977 Cond = Cmp; 4978 addTest = false; 4979 } 4980 } 4981 4982 if (addTest) { 4983 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4984 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4985 } 4986 4987 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4988 MVT::Flag); 4989 SmallVector<SDValue, 4> Ops; 4990 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4991 // condition is true. 4992 Ops.push_back(Op.getOperand(2)); 4993 Ops.push_back(Op.getOperand(1)); 4994 Ops.push_back(CC); 4995 Ops.push_back(Cond); 4996 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4997} 4998 4999SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) { 5000 bool addTest = true; 5001 SDValue Chain = Op.getOperand(0); 5002 SDValue Cond = Op.getOperand(1); 5003 SDValue Dest = Op.getOperand(2); 5004 SDValue CC; 5005 5006 if (Cond.getOpcode() == ISD::SETCC) 5007 Cond = LowerSETCC(Cond, DAG); 5008 5009 // If condition flag is set by a X86ISD::CMP, then use it as the condition 5010 // setting operand in place of the X86ISD::SETCC. 5011 if (Cond.getOpcode() == X86ISD::SETCC) { 5012 CC = Cond.getOperand(0); 5013 5014 SDValue Cmp = Cond.getOperand(1); 5015 unsigned Opc = Cmp.getOpcode(); 5016 if (Opc == X86ISD::CMP || 5017 Opc == X86ISD::COMI || 5018 Opc == X86ISD::UCOMI) { 5019 Cond = Cmp; 5020 addTest = false; 5021 } 5022 } 5023 5024 if (addTest) { 5025 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 5026 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 5027 } 5028 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 5029 Chain, Op.getOperand(2), CC, Cond); 5030} 5031 5032 5033// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 5034// Calls to _alloca is needed to probe the stack when allocating more than 4k 5035// bytes in one go. Touching the stack at 4K increments is necessary to ensure 5036// that the guard pages used by the OS virtual memory manager are allocated in 5037// correct sequence. 5038SDValue 5039X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 5040 SelectionDAG &DAG) { 5041 assert(Subtarget->isTargetCygMing() && 5042 "This should be used only on Cygwin/Mingw targets"); 5043 5044 // Get the inputs. 5045 SDValue Chain = Op.getOperand(0); 5046 SDValue Size = Op.getOperand(1); 5047 // FIXME: Ensure alignment here 5048 5049 SDValue Flag; 5050 5051 MVT IntPtr = getPointerTy(); 5052 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 5053 5054 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 5055 5056 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 5057 Flag = Chain.getValue(1); 5058 5059 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 5060 SDValue Ops[] = { Chain, 5061 DAG.getTargetExternalSymbol("_alloca", IntPtr), 5062 DAG.getRegister(X86::EAX, IntPtr), 5063 DAG.getRegister(X86StackPtr, SPTy), 5064 Flag }; 5065 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 5066 Flag = Chain.getValue(1); 5067 5068 Chain = DAG.getCALLSEQ_END(Chain, 5069 DAG.getIntPtrConstant(0), 5070 DAG.getIntPtrConstant(0), 5071 Flag); 5072 5073 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 5074 5075 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 5076 return DAG.getMergeValues(Ops1, 2); 5077} 5078 5079SDValue 5080X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 5081 SDValue Chain, 5082 SDValue Dst, SDValue Src, 5083 SDValue Size, unsigned Align, 5084 const Value *DstSV, uint64_t DstSVOff) { 5085 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5086 5087 /// If not DWORD aligned or size is more than the threshold, call the library. 5088 /// The libc version is likely to be faster for these cases. It can use the 5089 /// address value and run time information about the CPU. 5090 if ((Align & 3) != 0 || 5091 !ConstantSize || 5092 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 5093 SDValue InFlag(0, 0); 5094 5095 // Check to see if there is a specialized entry-point for memory zeroing. 5096 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 5097 if (const char *bzeroEntry = 5098 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 5099 MVT IntPtr = getPointerTy(); 5100 const Type *IntPtrTy = TD->getIntPtrType(); 5101 TargetLowering::ArgListTy Args; 5102 TargetLowering::ArgListEntry Entry; 5103 Entry.Node = Dst; 5104 Entry.Ty = IntPtrTy; 5105 Args.push_back(Entry); 5106 Entry.Node = Size; 5107 Args.push_back(Entry); 5108 std::pair<SDValue,SDValue> CallResult = 5109 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 5110 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 5111 Args, DAG); 5112 return CallResult.second; 5113 } 5114 5115 // Otherwise have the target-independent code call memset. 5116 return SDValue(); 5117 } 5118 5119 uint64_t SizeVal = ConstantSize->getValue(); 5120 SDValue InFlag(0, 0); 5121 MVT AVT; 5122 SDValue Count; 5123 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 5124 unsigned BytesLeft = 0; 5125 bool TwoRepStos = false; 5126 if (ValC) { 5127 unsigned ValReg; 5128 uint64_t Val = ValC->getValue() & 255; 5129 5130 // If the value is a constant, then we can potentially use larger sets. 5131 switch (Align & 3) { 5132 case 2: // WORD aligned 5133 AVT = MVT::i16; 5134 ValReg = X86::AX; 5135 Val = (Val << 8) | Val; 5136 break; 5137 case 0: // DWORD aligned 5138 AVT = MVT::i32; 5139 ValReg = X86::EAX; 5140 Val = (Val << 8) | Val; 5141 Val = (Val << 16) | Val; 5142 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 5143 AVT = MVT::i64; 5144 ValReg = X86::RAX; 5145 Val = (Val << 32) | Val; 5146 } 5147 break; 5148 default: // Byte aligned 5149 AVT = MVT::i8; 5150 ValReg = X86::AL; 5151 Count = DAG.getIntPtrConstant(SizeVal); 5152 break; 5153 } 5154 5155 if (AVT.bitsGT(MVT::i8)) { 5156 unsigned UBytes = AVT.getSizeInBits() / 8; 5157 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 5158 BytesLeft = SizeVal % UBytes; 5159 } 5160 5161 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 5162 InFlag); 5163 InFlag = Chain.getValue(1); 5164 } else { 5165 AVT = MVT::i8; 5166 Count = DAG.getIntPtrConstant(SizeVal); 5167 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 5168 InFlag = Chain.getValue(1); 5169 } 5170 5171 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5172 Count, InFlag); 5173 InFlag = Chain.getValue(1); 5174 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5175 Dst, InFlag); 5176 InFlag = Chain.getValue(1); 5177 5178 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5179 SmallVector<SDValue, 8> Ops; 5180 Ops.push_back(Chain); 5181 Ops.push_back(DAG.getValueType(AVT)); 5182 Ops.push_back(InFlag); 5183 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5184 5185 if (TwoRepStos) { 5186 InFlag = Chain.getValue(1); 5187 Count = Size; 5188 MVT CVT = Count.getValueType(); 5189 SDValue Left = DAG.getNode(ISD::AND, CVT, Count, 5190 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 5191 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 5192 Left, InFlag); 5193 InFlag = Chain.getValue(1); 5194 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5195 Ops.clear(); 5196 Ops.push_back(Chain); 5197 Ops.push_back(DAG.getValueType(MVT::i8)); 5198 Ops.push_back(InFlag); 5199 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5200 } else if (BytesLeft) { 5201 // Handle the last 1 - 7 bytes. 5202 unsigned Offset = SizeVal - BytesLeft; 5203 MVT AddrVT = Dst.getValueType(); 5204 MVT SizeVT = Size.getValueType(); 5205 5206 Chain = DAG.getMemset(Chain, 5207 DAG.getNode(ISD::ADD, AddrVT, Dst, 5208 DAG.getConstant(Offset, AddrVT)), 5209 Src, 5210 DAG.getConstant(BytesLeft, SizeVT), 5211 Align, DstSV, DstSVOff + Offset); 5212 } 5213 5214 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 5215 return Chain; 5216} 5217 5218SDValue 5219X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 5220 SDValue Chain, SDValue Dst, SDValue Src, 5221 SDValue Size, unsigned Align, 5222 bool AlwaysInline, 5223 const Value *DstSV, uint64_t DstSVOff, 5224 const Value *SrcSV, uint64_t SrcSVOff) { 5225 // This requires the copy size to be a constant, preferrably 5226 // within a subtarget-specific limit. 5227 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5228 if (!ConstantSize) 5229 return SDValue(); 5230 uint64_t SizeVal = ConstantSize->getValue(); 5231 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 5232 return SDValue(); 5233 5234 /// If not DWORD aligned, call the library. 5235 if ((Align & 3) != 0) 5236 return SDValue(); 5237 5238 // DWORD aligned 5239 MVT AVT = MVT::i32; 5240 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) // QWORD aligned 5241 AVT = MVT::i64; 5242 5243 unsigned UBytes = AVT.getSizeInBits() / 8; 5244 unsigned CountVal = SizeVal / UBytes; 5245 SDValue Count = DAG.getIntPtrConstant(CountVal); 5246 unsigned BytesLeft = SizeVal % UBytes; 5247 5248 SDValue InFlag(0, 0); 5249 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5250 Count, InFlag); 5251 InFlag = Chain.getValue(1); 5252 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5253 Dst, InFlag); 5254 InFlag = Chain.getValue(1); 5255 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 5256 Src, InFlag); 5257 InFlag = Chain.getValue(1); 5258 5259 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5260 SmallVector<SDValue, 8> Ops; 5261 Ops.push_back(Chain); 5262 Ops.push_back(DAG.getValueType(AVT)); 5263 Ops.push_back(InFlag); 5264 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5265 5266 SmallVector<SDValue, 4> Results; 5267 Results.push_back(RepMovs); 5268 if (BytesLeft) { 5269 // Handle the last 1 - 7 bytes. 5270 unsigned Offset = SizeVal - BytesLeft; 5271 MVT DstVT = Dst.getValueType(); 5272 MVT SrcVT = Src.getValueType(); 5273 MVT SizeVT = Size.getValueType(); 5274 Results.push_back(DAG.getMemcpy(Chain, 5275 DAG.getNode(ISD::ADD, DstVT, Dst, 5276 DAG.getConstant(Offset, DstVT)), 5277 DAG.getNode(ISD::ADD, SrcVT, Src, 5278 DAG.getConstant(Offset, SrcVT)), 5279 DAG.getConstant(BytesLeft, SizeVT), 5280 Align, AlwaysInline, 5281 DstSV, DstSVOff + Offset, 5282 SrcSV, SrcSVOff + Offset)); 5283 } 5284 5285 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5286} 5287 5288/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5289SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5290 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5291 SDValue TheChain = N->getOperand(0); 5292 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5293 if (Subtarget->is64Bit()) { 5294 SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5295 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5296 MVT::i64, rax.getValue(2)); 5297 SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5298 DAG.getConstant(32, MVT::i8)); 5299 SDValue Ops[] = { 5300 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5301 }; 5302 5303 return DAG.getMergeValues(Ops, 2).getNode(); 5304 } 5305 5306 SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5307 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5308 MVT::i32, eax.getValue(2)); 5309 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5310 SDValue Ops[] = { eax, edx }; 5311 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5312 5313 // Use a MERGE_VALUES to return the value and chain. 5314 Ops[1] = edx.getValue(1); 5315 return DAG.getMergeValues(Ops, 2).getNode(); 5316} 5317 5318SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) { 5319 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5320 5321 if (!Subtarget->is64Bit()) { 5322 // vastart just stores the address of the VarArgsFrameIndex slot into the 5323 // memory location argument. 5324 SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5325 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5326 } 5327 5328 // __va_list_tag: 5329 // gp_offset (0 - 6 * 8) 5330 // fp_offset (48 - 48 + 8 * 16) 5331 // overflow_arg_area (point to parameters coming in memory). 5332 // reg_save_area 5333 SmallVector<SDValue, 8> MemOps; 5334 SDValue FIN = Op.getOperand(1); 5335 // Store gp_offset 5336 SDValue Store = DAG.getStore(Op.getOperand(0), 5337 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5338 FIN, SV, 0); 5339 MemOps.push_back(Store); 5340 5341 // Store fp_offset 5342 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5343 Store = DAG.getStore(Op.getOperand(0), 5344 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5345 FIN, SV, 0); 5346 MemOps.push_back(Store); 5347 5348 // Store ptr to overflow_arg_area 5349 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5350 SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5351 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5352 MemOps.push_back(Store); 5353 5354 // Store ptr to reg_save_area. 5355 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5356 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5357 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5358 MemOps.push_back(Store); 5359 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5360} 5361 5362SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) { 5363 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5364 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5365 SDValue Chain = Op.getOperand(0); 5366 SDValue SrcPtr = Op.getOperand(1); 5367 SDValue SrcSV = Op.getOperand(2); 5368 5369 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5370 abort(); 5371 return SDValue(); 5372} 5373 5374SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) { 5375 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5376 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5377 SDValue Chain = Op.getOperand(0); 5378 SDValue DstPtr = Op.getOperand(1); 5379 SDValue SrcPtr = Op.getOperand(2); 5380 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5381 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5382 5383 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5384 DAG.getIntPtrConstant(24), 8, false, 5385 DstSV, 0, SrcSV, 0); 5386} 5387 5388SDValue 5389X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { 5390 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5391 switch (IntNo) { 5392 default: return SDValue(); // Don't custom lower most intrinsics. 5393 // Comparison intrinsics. 5394 case Intrinsic::x86_sse_comieq_ss: 5395 case Intrinsic::x86_sse_comilt_ss: 5396 case Intrinsic::x86_sse_comile_ss: 5397 case Intrinsic::x86_sse_comigt_ss: 5398 case Intrinsic::x86_sse_comige_ss: 5399 case Intrinsic::x86_sse_comineq_ss: 5400 case Intrinsic::x86_sse_ucomieq_ss: 5401 case Intrinsic::x86_sse_ucomilt_ss: 5402 case Intrinsic::x86_sse_ucomile_ss: 5403 case Intrinsic::x86_sse_ucomigt_ss: 5404 case Intrinsic::x86_sse_ucomige_ss: 5405 case Intrinsic::x86_sse_ucomineq_ss: 5406 case Intrinsic::x86_sse2_comieq_sd: 5407 case Intrinsic::x86_sse2_comilt_sd: 5408 case Intrinsic::x86_sse2_comile_sd: 5409 case Intrinsic::x86_sse2_comigt_sd: 5410 case Intrinsic::x86_sse2_comige_sd: 5411 case Intrinsic::x86_sse2_comineq_sd: 5412 case Intrinsic::x86_sse2_ucomieq_sd: 5413 case Intrinsic::x86_sse2_ucomilt_sd: 5414 case Intrinsic::x86_sse2_ucomile_sd: 5415 case Intrinsic::x86_sse2_ucomigt_sd: 5416 case Intrinsic::x86_sse2_ucomige_sd: 5417 case Intrinsic::x86_sse2_ucomineq_sd: { 5418 unsigned Opc = 0; 5419 ISD::CondCode CC = ISD::SETCC_INVALID; 5420 switch (IntNo) { 5421 default: break; 5422 case Intrinsic::x86_sse_comieq_ss: 5423 case Intrinsic::x86_sse2_comieq_sd: 5424 Opc = X86ISD::COMI; 5425 CC = ISD::SETEQ; 5426 break; 5427 case Intrinsic::x86_sse_comilt_ss: 5428 case Intrinsic::x86_sse2_comilt_sd: 5429 Opc = X86ISD::COMI; 5430 CC = ISD::SETLT; 5431 break; 5432 case Intrinsic::x86_sse_comile_ss: 5433 case Intrinsic::x86_sse2_comile_sd: 5434 Opc = X86ISD::COMI; 5435 CC = ISD::SETLE; 5436 break; 5437 case Intrinsic::x86_sse_comigt_ss: 5438 case Intrinsic::x86_sse2_comigt_sd: 5439 Opc = X86ISD::COMI; 5440 CC = ISD::SETGT; 5441 break; 5442 case Intrinsic::x86_sse_comige_ss: 5443 case Intrinsic::x86_sse2_comige_sd: 5444 Opc = X86ISD::COMI; 5445 CC = ISD::SETGE; 5446 break; 5447 case Intrinsic::x86_sse_comineq_ss: 5448 case Intrinsic::x86_sse2_comineq_sd: 5449 Opc = X86ISD::COMI; 5450 CC = ISD::SETNE; 5451 break; 5452 case Intrinsic::x86_sse_ucomieq_ss: 5453 case Intrinsic::x86_sse2_ucomieq_sd: 5454 Opc = X86ISD::UCOMI; 5455 CC = ISD::SETEQ; 5456 break; 5457 case Intrinsic::x86_sse_ucomilt_ss: 5458 case Intrinsic::x86_sse2_ucomilt_sd: 5459 Opc = X86ISD::UCOMI; 5460 CC = ISD::SETLT; 5461 break; 5462 case Intrinsic::x86_sse_ucomile_ss: 5463 case Intrinsic::x86_sse2_ucomile_sd: 5464 Opc = X86ISD::UCOMI; 5465 CC = ISD::SETLE; 5466 break; 5467 case Intrinsic::x86_sse_ucomigt_ss: 5468 case Intrinsic::x86_sse2_ucomigt_sd: 5469 Opc = X86ISD::UCOMI; 5470 CC = ISD::SETGT; 5471 break; 5472 case Intrinsic::x86_sse_ucomige_ss: 5473 case Intrinsic::x86_sse2_ucomige_sd: 5474 Opc = X86ISD::UCOMI; 5475 CC = ISD::SETGE; 5476 break; 5477 case Intrinsic::x86_sse_ucomineq_ss: 5478 case Intrinsic::x86_sse2_ucomineq_sd: 5479 Opc = X86ISD::UCOMI; 5480 CC = ISD::SETNE; 5481 break; 5482 } 5483 5484 unsigned X86CC; 5485 SDValue LHS = Op.getOperand(1); 5486 SDValue RHS = Op.getOperand(2); 5487 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5488 5489 SDValue Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5490 SDValue SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5491 DAG.getConstant(X86CC, MVT::i8), Cond); 5492 return DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, SetCC); 5493 } 5494 5495 // Fix vector shift instructions where the last operand is a non-immediate 5496 // i32 value. 5497 case Intrinsic::x86_sse2_pslli_w: 5498 case Intrinsic::x86_sse2_pslli_d: 5499 case Intrinsic::x86_sse2_pslli_q: 5500 case Intrinsic::x86_sse2_psrli_w: 5501 case Intrinsic::x86_sse2_psrli_d: 5502 case Intrinsic::x86_sse2_psrli_q: 5503 case Intrinsic::x86_sse2_psrai_w: 5504 case Intrinsic::x86_sse2_psrai_d: 5505 case Intrinsic::x86_mmx_pslli_w: 5506 case Intrinsic::x86_mmx_pslli_d: 5507 case Intrinsic::x86_mmx_pslli_q: 5508 case Intrinsic::x86_mmx_psrli_w: 5509 case Intrinsic::x86_mmx_psrli_d: 5510 case Intrinsic::x86_mmx_psrli_q: 5511 case Intrinsic::x86_mmx_psrai_w: 5512 case Intrinsic::x86_mmx_psrai_d: { 5513 SDValue ShAmt = Op.getOperand(2); 5514 if (isa<ConstantSDNode>(ShAmt)) 5515 return SDValue(); 5516 5517 unsigned NewIntNo = 0; 5518 MVT ShAmtVT = MVT::v4i32; 5519 switch (IntNo) { 5520 case Intrinsic::x86_sse2_pslli_w: 5521 NewIntNo = Intrinsic::x86_sse2_psll_w; 5522 break; 5523 case Intrinsic::x86_sse2_pslli_d: 5524 NewIntNo = Intrinsic::x86_sse2_psll_d; 5525 break; 5526 case Intrinsic::x86_sse2_pslli_q: 5527 NewIntNo = Intrinsic::x86_sse2_psll_q; 5528 break; 5529 case Intrinsic::x86_sse2_psrli_w: 5530 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5531 break; 5532 case Intrinsic::x86_sse2_psrli_d: 5533 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5534 break; 5535 case Intrinsic::x86_sse2_psrli_q: 5536 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5537 break; 5538 case Intrinsic::x86_sse2_psrai_w: 5539 NewIntNo = Intrinsic::x86_sse2_psra_w; 5540 break; 5541 case Intrinsic::x86_sse2_psrai_d: 5542 NewIntNo = Intrinsic::x86_sse2_psra_d; 5543 break; 5544 default: { 5545 ShAmtVT = MVT::v2i32; 5546 switch (IntNo) { 5547 case Intrinsic::x86_mmx_pslli_w: 5548 NewIntNo = Intrinsic::x86_mmx_psll_w; 5549 break; 5550 case Intrinsic::x86_mmx_pslli_d: 5551 NewIntNo = Intrinsic::x86_mmx_psll_d; 5552 break; 5553 case Intrinsic::x86_mmx_pslli_q: 5554 NewIntNo = Intrinsic::x86_mmx_psll_q; 5555 break; 5556 case Intrinsic::x86_mmx_psrli_w: 5557 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5558 break; 5559 case Intrinsic::x86_mmx_psrli_d: 5560 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5561 break; 5562 case Intrinsic::x86_mmx_psrli_q: 5563 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5564 break; 5565 case Intrinsic::x86_mmx_psrai_w: 5566 NewIntNo = Intrinsic::x86_mmx_psra_w; 5567 break; 5568 case Intrinsic::x86_mmx_psrai_d: 5569 NewIntNo = Intrinsic::x86_mmx_psra_d; 5570 break; 5571 default: abort(); // Can't reach here. 5572 } 5573 break; 5574 } 5575 } 5576 MVT VT = Op.getValueType(); 5577 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5578 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5579 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5580 DAG.getConstant(NewIntNo, MVT::i32), 5581 Op.getOperand(1), ShAmt); 5582 } 5583 } 5584} 5585 5586SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) { 5587 // Depths > 0 not supported yet! 5588 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5589 return SDValue(); 5590 5591 // Just load the return address 5592 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5593 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5594} 5595 5596SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) { 5597 // Depths > 0 not supported yet! 5598 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5599 return SDValue(); 5600 5601 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5602 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5603 DAG.getIntPtrConstant(TD->getPointerSize())); 5604} 5605 5606SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 5607 SelectionDAG &DAG) { 5608 return DAG.getIntPtrConstant(2*TD->getPointerSize()); 5609} 5610 5611SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) 5612{ 5613 MachineFunction &MF = DAG.getMachineFunction(); 5614 SDValue Chain = Op.getOperand(0); 5615 SDValue Offset = Op.getOperand(1); 5616 SDValue Handler = Op.getOperand(2); 5617 5618 SDValue Frame = DAG.getRegister(Subtarget->is64Bit() ? X86::RBP : X86::EBP, 5619 getPointerTy()); 5620 unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX); 5621 5622 SDValue StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5623 DAG.getIntPtrConstant(-TD->getPointerSize())); 5624 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5625 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5626 Chain = DAG.getCopyToReg(Chain, StoreAddrReg, StoreAddr); 5627 MF.getRegInfo().addLiveOut(StoreAddrReg); 5628 5629 return DAG.getNode(X86ISD::EH_RETURN, 5630 MVT::Other, 5631 Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); 5632} 5633 5634SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, 5635 SelectionDAG &DAG) { 5636 SDValue Root = Op.getOperand(0); 5637 SDValue Trmp = Op.getOperand(1); // trampoline 5638 SDValue FPtr = Op.getOperand(2); // nested function 5639 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 5640 5641 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5642 5643 const X86InstrInfo *TII = 5644 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5645 5646 if (Subtarget->is64Bit()) { 5647 SDValue OutChains[6]; 5648 5649 // Large code-model. 5650 5651 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5652 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5653 5654 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5655 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5656 5657 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5658 5659 // Load the pointer to the nested function into R11. 5660 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5661 SDValue Addr = Trmp; 5662 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5663 TrmpAddr, 0); 5664 5665 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5666 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5667 5668 // Load the 'nest' parameter value into R10. 5669 // R10 is specified in X86CallingConv.td 5670 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5671 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5672 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5673 TrmpAddr, 10); 5674 5675 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5676 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5677 5678 // Jump to the nested function. 5679 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5680 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5681 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5682 TrmpAddr, 20); 5683 5684 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5685 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5686 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5687 TrmpAddr, 22); 5688 5689 SDValue Ops[] = 5690 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5691 return DAG.getMergeValues(Ops, 2); 5692 } else { 5693 const Function *Func = 5694 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5695 unsigned CC = Func->getCallingConv(); 5696 unsigned NestReg; 5697 5698 switch (CC) { 5699 default: 5700 assert(0 && "Unsupported calling convention"); 5701 case CallingConv::C: 5702 case CallingConv::X86_StdCall: { 5703 // Pass 'nest' parameter in ECX. 5704 // Must be kept in sync with X86CallingConv.td 5705 NestReg = X86::ECX; 5706 5707 // Check that ECX wasn't needed by an 'inreg' parameter. 5708 const FunctionType *FTy = Func->getFunctionType(); 5709 const PAListPtr &Attrs = Func->getParamAttrs(); 5710 5711 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5712 unsigned InRegCount = 0; 5713 unsigned Idx = 1; 5714 5715 for (FunctionType::param_iterator I = FTy->param_begin(), 5716 E = FTy->param_end(); I != E; ++I, ++Idx) 5717 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5718 // FIXME: should only count parameters that are lowered to integers. 5719 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32; 5720 5721 if (InRegCount > 2) { 5722 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5723 abort(); 5724 } 5725 } 5726 break; 5727 } 5728 case CallingConv::X86_FastCall: 5729 case CallingConv::Fast: 5730 // Pass 'nest' parameter in EAX. 5731 // Must be kept in sync with X86CallingConv.td 5732 NestReg = X86::EAX; 5733 break; 5734 } 5735 5736 SDValue OutChains[4]; 5737 SDValue Addr, Disp; 5738 5739 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5740 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5741 5742 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5743 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5744 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5745 Trmp, TrmpAddr, 0); 5746 5747 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5748 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5749 5750 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5751 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5752 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5753 TrmpAddr, 5, false, 1); 5754 5755 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5756 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5757 5758 SDValue Ops[] = 5759 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5760 return DAG.getMergeValues(Ops, 2); 5761 } 5762} 5763 5764SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) { 5765 /* 5766 The rounding mode is in bits 11:10 of FPSR, and has the following 5767 settings: 5768 00 Round to nearest 5769 01 Round to -inf 5770 10 Round to +inf 5771 11 Round to 0 5772 5773 FLT_ROUNDS, on the other hand, expects the following: 5774 -1 Undefined 5775 0 Round to 0 5776 1 Round to nearest 5777 2 Round to +inf 5778 3 Round to -inf 5779 5780 To perform the conversion, we do: 5781 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5782 */ 5783 5784 MachineFunction &MF = DAG.getMachineFunction(); 5785 const TargetMachine &TM = MF.getTarget(); 5786 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5787 unsigned StackAlignment = TFI.getStackAlignment(); 5788 MVT VT = Op.getValueType(); 5789 5790 // Save FP Control Word to stack slot 5791 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5792 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5793 5794 SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5795 DAG.getEntryNode(), StackSlot); 5796 5797 // Load FP Control Word from stack slot 5798 SDValue CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5799 5800 // Transform as necessary 5801 SDValue CWD1 = 5802 DAG.getNode(ISD::SRL, MVT::i16, 5803 DAG.getNode(ISD::AND, MVT::i16, 5804 CWD, DAG.getConstant(0x800, MVT::i16)), 5805 DAG.getConstant(11, MVT::i8)); 5806 SDValue CWD2 = 5807 DAG.getNode(ISD::SRL, MVT::i16, 5808 DAG.getNode(ISD::AND, MVT::i16, 5809 CWD, DAG.getConstant(0x400, MVT::i16)), 5810 DAG.getConstant(9, MVT::i8)); 5811 5812 SDValue RetVal = 5813 DAG.getNode(ISD::AND, MVT::i16, 5814 DAG.getNode(ISD::ADD, MVT::i16, 5815 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5816 DAG.getConstant(1, MVT::i16)), 5817 DAG.getConstant(3, MVT::i16)); 5818 5819 5820 return DAG.getNode((VT.getSizeInBits() < 16 ? 5821 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5822} 5823 5824SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) { 5825 MVT VT = Op.getValueType(); 5826 MVT OpVT = VT; 5827 unsigned NumBits = VT.getSizeInBits(); 5828 5829 Op = Op.getOperand(0); 5830 if (VT == MVT::i8) { 5831 // Zero extend to i32 since there is not an i8 bsr. 5832 OpVT = MVT::i32; 5833 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5834 } 5835 5836 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5837 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5838 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5839 5840 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5841 SmallVector<SDValue, 4> Ops; 5842 Ops.push_back(Op); 5843 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5844 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5845 Ops.push_back(Op.getValue(1)); 5846 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5847 5848 // Finally xor with NumBits-1. 5849 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5850 5851 if (VT == MVT::i8) 5852 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5853 return Op; 5854} 5855 5856SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) { 5857 MVT VT = Op.getValueType(); 5858 MVT OpVT = VT; 5859 unsigned NumBits = VT.getSizeInBits(); 5860 5861 Op = Op.getOperand(0); 5862 if (VT == MVT::i8) { 5863 OpVT = MVT::i32; 5864 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5865 } 5866 5867 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5868 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5869 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5870 5871 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5872 SmallVector<SDValue, 4> Ops; 5873 Ops.push_back(Op); 5874 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5875 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5876 Ops.push_back(Op.getValue(1)); 5877 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5878 5879 if (VT == MVT::i8) 5880 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5881 return Op; 5882} 5883 5884SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) { 5885 MVT T = Op.getValueType(); 5886 unsigned Reg = 0; 5887 unsigned size = 0; 5888 switch(T.getSimpleVT()) { 5889 default: 5890 assert(false && "Invalid value type!"); 5891 case MVT::i8: Reg = X86::AL; size = 1; break; 5892 case MVT::i16: Reg = X86::AX; size = 2; break; 5893 case MVT::i32: Reg = X86::EAX; size = 4; break; 5894 case MVT::i64: 5895 if (Subtarget->is64Bit()) { 5896 Reg = X86::RAX; size = 8; 5897 } else //Should go away when LowerType stuff lands 5898 return SDValue(ExpandATOMIC_CMP_SWAP(Op.getNode(), DAG), 0); 5899 break; 5900 }; 5901 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5902 Op.getOperand(2), SDValue()); 5903 SDValue Ops[] = { cpIn.getValue(0), 5904 Op.getOperand(1), 5905 Op.getOperand(3), 5906 DAG.getTargetConstant(size, MVT::i8), 5907 cpIn.getValue(1) }; 5908 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5909 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5910 SDValue cpOut = 5911 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5912 return cpOut; 5913} 5914 5915SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, 5916 SelectionDAG &DAG) { 5917 MVT T = Op->getValueType(0); 5918 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5919 SDValue cpInL, cpInH; 5920 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5921 DAG.getConstant(0, MVT::i32)); 5922 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5923 DAG.getConstant(1, MVT::i32)); 5924 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5925 cpInL, SDValue()); 5926 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5927 cpInH, cpInL.getValue(1)); 5928 SDValue swapInL, swapInH; 5929 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5930 DAG.getConstant(0, MVT::i32)); 5931 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5932 DAG.getConstant(1, MVT::i32)); 5933 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5934 swapInL, cpInH.getValue(1)); 5935 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5936 swapInH, swapInL.getValue(1)); 5937 SDValue Ops[] = { swapInH.getValue(0), 5938 Op->getOperand(1), 5939 swapInH.getValue(1)}; 5940 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5941 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5942 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5943 Result.getValue(1)); 5944 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5945 cpOutL.getValue(2)); 5946 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5947 SDValue ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5948 SDValue Vals[2] = { ResultVal, cpOutH.getValue(1) }; 5949 return DAG.getMergeValues(Vals, 2).getNode(); 5950} 5951 5952SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, 5953 SelectionDAG &DAG) { 5954 MVT T = Op->getValueType(0); 5955 SDValue negOp = DAG.getNode(ISD::SUB, T, 5956 DAG.getConstant(0, T), Op->getOperand(2)); 5957 return DAG.getAtomic((T==MVT::i8 ? ISD::ATOMIC_LOAD_ADD_8: 5958 T==MVT::i16 ? ISD::ATOMIC_LOAD_ADD_16: 5959 T==MVT::i32 ? ISD::ATOMIC_LOAD_ADD_32: 5960 T==MVT::i64 ? ISD::ATOMIC_LOAD_ADD_64: 0), 5961 Op->getOperand(0), Op->getOperand(1), negOp, 5962 cast<AtomicSDNode>(Op)->getSrcValue(), 5963 cast<AtomicSDNode>(Op)->getAlignment()).getNode(); 5964} 5965 5966/// LowerOperation - Provide custom lowering hooks for some operations. 5967/// 5968SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { 5969 switch (Op.getOpcode()) { 5970 default: assert(0 && "Should not custom lower this!"); 5971 case ISD::ATOMIC_CMP_SWAP_8: return LowerCMP_SWAP(Op,DAG); 5972 case ISD::ATOMIC_CMP_SWAP_16: return LowerCMP_SWAP(Op,DAG); 5973 case ISD::ATOMIC_CMP_SWAP_32: return LowerCMP_SWAP(Op,DAG); 5974 case ISD::ATOMIC_CMP_SWAP_64: return LowerCMP_SWAP(Op,DAG); 5975 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5976 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5977 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5978 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5979 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5980 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5981 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5982 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5983 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5984 case ISD::SHL_PARTS: 5985 case ISD::SRA_PARTS: 5986 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5987 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5988 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5989 case ISD::FABS: return LowerFABS(Op, DAG); 5990 case ISD::FNEG: return LowerFNEG(Op, DAG); 5991 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5992 case ISD::SETCC: return LowerSETCC(Op, DAG); 5993 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 5994 case ISD::SELECT: return LowerSELECT(Op, DAG); 5995 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5996 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5997 case ISD::CALL: return LowerCALL(Op, DAG); 5998 case ISD::RET: return LowerRET(Op, DAG); 5999 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 6000 case ISD::VASTART: return LowerVASTART(Op, DAG); 6001 case ISD::VAARG: return LowerVAARG(Op, DAG); 6002 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 6003 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 6004 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 6005 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 6006 case ISD::FRAME_TO_ARGS_OFFSET: 6007 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 6008 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 6009 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 6010 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 6011 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 6012 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 6013 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 6014 6015 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 6016 case ISD::READCYCLECOUNTER: 6017 return SDValue(ExpandREADCYCLECOUNTER(Op.getNode(), DAG), 0); 6018 } 6019} 6020 6021/// ReplaceNodeResults - Replace a node with an illegal result type 6022/// with a new node built out of custom code. 6023SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) { 6024 switch (N->getOpcode()) { 6025 default: assert(0 && "Should not custom lower this!"); 6026 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 6027 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 6028 case ISD::ATOMIC_CMP_SWAP_64: return ExpandATOMIC_CMP_SWAP(N, DAG); 6029 case ISD::ATOMIC_LOAD_SUB_8: return ExpandATOMIC_LOAD_SUB(N,DAG); 6030 case ISD::ATOMIC_LOAD_SUB_16: return ExpandATOMIC_LOAD_SUB(N,DAG); 6031 case ISD::ATOMIC_LOAD_SUB_32: return ExpandATOMIC_LOAD_SUB(N,DAG); 6032 case ISD::ATOMIC_LOAD_SUB_64: return ExpandATOMIC_LOAD_SUB(N,DAG); 6033 } 6034} 6035 6036const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 6037 switch (Opcode) { 6038 default: return NULL; 6039 case X86ISD::BSF: return "X86ISD::BSF"; 6040 case X86ISD::BSR: return "X86ISD::BSR"; 6041 case X86ISD::SHLD: return "X86ISD::SHLD"; 6042 case X86ISD::SHRD: return "X86ISD::SHRD"; 6043 case X86ISD::FAND: return "X86ISD::FAND"; 6044 case X86ISD::FOR: return "X86ISD::FOR"; 6045 case X86ISD::FXOR: return "X86ISD::FXOR"; 6046 case X86ISD::FSRL: return "X86ISD::FSRL"; 6047 case X86ISD::FILD: return "X86ISD::FILD"; 6048 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 6049 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 6050 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 6051 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 6052 case X86ISD::FLD: return "X86ISD::FLD"; 6053 case X86ISD::FST: return "X86ISD::FST"; 6054 case X86ISD::CALL: return "X86ISD::CALL"; 6055 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 6056 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 6057 case X86ISD::CMP: return "X86ISD::CMP"; 6058 case X86ISD::COMI: return "X86ISD::COMI"; 6059 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 6060 case X86ISD::SETCC: return "X86ISD::SETCC"; 6061 case X86ISD::CMOV: return "X86ISD::CMOV"; 6062 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 6063 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 6064 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 6065 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 6066 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 6067 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 6068 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 6069 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 6070 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 6071 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 6072 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 6073 case X86ISD::FMAX: return "X86ISD::FMAX"; 6074 case X86ISD::FMIN: return "X86ISD::FMIN"; 6075 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 6076 case X86ISD::FRCP: return "X86ISD::FRCP"; 6077 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 6078 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 6079 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 6080 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 6081 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 6082 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 6083 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 6084 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 6085 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 6086 case X86ISD::VSHL: return "X86ISD::VSHL"; 6087 case X86ISD::VSRL: return "X86ISD::VSRL"; 6088 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 6089 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 6090 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 6091 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 6092 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 6093 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 6094 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 6095 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 6096 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 6097 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 6098 } 6099} 6100 6101// isLegalAddressingMode - Return true if the addressing mode represented 6102// by AM is legal for this target, for a load/store of the specified type. 6103bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 6104 const Type *Ty) const { 6105 // X86 supports extremely general addressing modes. 6106 6107 // X86 allows a sign-extended 32-bit immediate field as a displacement. 6108 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 6109 return false; 6110 6111 if (AM.BaseGV) { 6112 // We can only fold this if we don't need an extra load. 6113 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 6114 return false; 6115 6116 // X86-64 only supports addr of globals in small code model. 6117 if (Subtarget->is64Bit()) { 6118 if (getTargetMachine().getCodeModel() != CodeModel::Small) 6119 return false; 6120 // If lower 4G is not available, then we must use rip-relative addressing. 6121 if (AM.BaseOffs || AM.Scale > 1) 6122 return false; 6123 } 6124 } 6125 6126 switch (AM.Scale) { 6127 case 0: 6128 case 1: 6129 case 2: 6130 case 4: 6131 case 8: 6132 // These scales always work. 6133 break; 6134 case 3: 6135 case 5: 6136 case 9: 6137 // These scales are formed with basereg+scalereg. Only accept if there is 6138 // no basereg yet. 6139 if (AM.HasBaseReg) 6140 return false; 6141 break; 6142 default: // Other stuff never works. 6143 return false; 6144 } 6145 6146 return true; 6147} 6148 6149 6150bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 6151 if (!Ty1->isInteger() || !Ty2->isInteger()) 6152 return false; 6153 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6154 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6155 if (NumBits1 <= NumBits2) 6156 return false; 6157 return Subtarget->is64Bit() || NumBits1 < 64; 6158} 6159 6160bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 6161 if (!VT1.isInteger() || !VT2.isInteger()) 6162 return false; 6163 unsigned NumBits1 = VT1.getSizeInBits(); 6164 unsigned NumBits2 = VT2.getSizeInBits(); 6165 if (NumBits1 <= NumBits2) 6166 return false; 6167 return Subtarget->is64Bit() || NumBits1 < 64; 6168} 6169 6170/// isShuffleMaskLegal - Targets can use this to indicate that they only 6171/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6172/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6173/// are assumed to be legal. 6174bool 6175X86TargetLowering::isShuffleMaskLegal(SDValue Mask, MVT VT) const { 6176 // Only do shuffles on 128-bit vector types for now. 6177 if (VT.getSizeInBits() == 64) return false; 6178 return (Mask.getNode()->getNumOperands() <= 4 || 6179 isIdentityMask(Mask.getNode()) || 6180 isIdentityMask(Mask.getNode(), true) || 6181 isSplatMask(Mask.getNode()) || 6182 isPSHUFHW_PSHUFLWMask(Mask.getNode()) || 6183 X86::isUNPCKLMask(Mask.getNode()) || 6184 X86::isUNPCKHMask(Mask.getNode()) || 6185 X86::isUNPCKL_v_undef_Mask(Mask.getNode()) || 6186 X86::isUNPCKH_v_undef_Mask(Mask.getNode())); 6187} 6188 6189bool 6190X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, 6191 MVT EVT, SelectionDAG &DAG) const { 6192 unsigned NumElts = BVOps.size(); 6193 // Only do shuffles on 128-bit vector types for now. 6194 if (EVT.getSizeInBits() * NumElts == 64) return false; 6195 if (NumElts == 2) return true; 6196 if (NumElts == 4) { 6197 return (isMOVLMask(&BVOps[0], 4) || 6198 isCommutedMOVL(&BVOps[0], 4, true) || 6199 isSHUFPMask(&BVOps[0], 4) || 6200 isCommutedSHUFP(&BVOps[0], 4)); 6201 } 6202 return false; 6203} 6204 6205//===----------------------------------------------------------------------===// 6206// X86 Scheduler Hooks 6207//===----------------------------------------------------------------------===// 6208 6209// private utility function 6210MachineBasicBlock * 6211X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 6212 MachineBasicBlock *MBB, 6213 unsigned regOpc, 6214 unsigned immOpc, 6215 unsigned LoadOpc, 6216 unsigned CXchgOpc, 6217 unsigned copyOpc, 6218 unsigned notOpc, 6219 unsigned EAXreg, 6220 TargetRegisterClass *RC, 6221 bool invSrc) { 6222 // For the atomic bitwise operator, we generate 6223 // thisMBB: 6224 // newMBB: 6225 // ld t1 = [bitinstr.addr] 6226 // op t2 = t1, [bitinstr.val] 6227 // mov EAX = t1 6228 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6229 // bz newMBB 6230 // fallthrough -->nextMBB 6231 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6232 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6233 MachineFunction::iterator MBBIter = MBB; 6234 ++MBBIter; 6235 6236 /// First build the CFG 6237 MachineFunction *F = MBB->getParent(); 6238 MachineBasicBlock *thisMBB = MBB; 6239 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6240 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6241 F->insert(MBBIter, newMBB); 6242 F->insert(MBBIter, nextMBB); 6243 6244 // Move all successors to thisMBB to nextMBB 6245 nextMBB->transferSuccessors(thisMBB); 6246 6247 // Update thisMBB to fall through to newMBB 6248 thisMBB->addSuccessor(newMBB); 6249 6250 // newMBB jumps to itself and fall through to nextMBB 6251 newMBB->addSuccessor(nextMBB); 6252 newMBB->addSuccessor(newMBB); 6253 6254 // Insert instructions into newMBB based on incoming instruction 6255 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 6256 MachineOperand& destOper = bInstr->getOperand(0); 6257 MachineOperand* argOpers[6]; 6258 int numArgs = bInstr->getNumOperands() - 1; 6259 for (int i=0; i < numArgs; ++i) 6260 argOpers[i] = &bInstr->getOperand(i+1); 6261 6262 // x86 address has 4 operands: base, index, scale, and displacement 6263 int lastAddrIndx = 3; // [0,3] 6264 int valArgIndx = 4; 6265 6266 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 6267 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(LoadOpc), t1); 6268 for (int i=0; i <= lastAddrIndx; ++i) 6269 (*MIB).addOperand(*argOpers[i]); 6270 6271 unsigned tt = F->getRegInfo().createVirtualRegister(RC); 6272 if (invSrc) { 6273 MIB = BuildMI(newMBB, TII->get(notOpc), tt).addReg(t1); 6274 } 6275 else 6276 tt = t1; 6277 6278 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 6279 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6280 && "invalid operand"); 6281 if (argOpers[valArgIndx]->isReg()) 6282 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6283 else 6284 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6285 MIB.addReg(tt); 6286 (*MIB).addOperand(*argOpers[valArgIndx]); 6287 6288 MIB = BuildMI(newMBB, TII->get(copyOpc), EAXreg); 6289 MIB.addReg(t1); 6290 6291 MIB = BuildMI(newMBB, TII->get(CXchgOpc)); 6292 for (int i=0; i <= lastAddrIndx; ++i) 6293 (*MIB).addOperand(*argOpers[i]); 6294 MIB.addReg(t2); 6295 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6296 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 6297 6298 MIB = BuildMI(newMBB, TII->get(copyOpc), destOper.getReg()); 6299 MIB.addReg(EAXreg); 6300 6301 // insert branch 6302 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6303 6304 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 6305 return nextMBB; 6306} 6307 6308// private utility function 6309MachineBasicBlock * 6310X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6311 MachineBasicBlock *MBB, 6312 unsigned cmovOpc) { 6313 // For the atomic min/max operator, we generate 6314 // thisMBB: 6315 // newMBB: 6316 // ld t1 = [min/max.addr] 6317 // mov t2 = [min/max.val] 6318 // cmp t1, t2 6319 // cmov[cond] t2 = t1 6320 // mov EAX = t1 6321 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6322 // bz newMBB 6323 // fallthrough -->nextMBB 6324 // 6325 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6326 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6327 MachineFunction::iterator MBBIter = MBB; 6328 ++MBBIter; 6329 6330 /// First build the CFG 6331 MachineFunction *F = MBB->getParent(); 6332 MachineBasicBlock *thisMBB = MBB; 6333 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6334 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6335 F->insert(MBBIter, newMBB); 6336 F->insert(MBBIter, nextMBB); 6337 6338 // Move all successors to thisMBB to nextMBB 6339 nextMBB->transferSuccessors(thisMBB); 6340 6341 // Update thisMBB to fall through to newMBB 6342 thisMBB->addSuccessor(newMBB); 6343 6344 // newMBB jumps to newMBB and fall through to nextMBB 6345 newMBB->addSuccessor(nextMBB); 6346 newMBB->addSuccessor(newMBB); 6347 6348 // Insert instructions into newMBB based on incoming instruction 6349 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6350 MachineOperand& destOper = mInstr->getOperand(0); 6351 MachineOperand* argOpers[6]; 6352 int numArgs = mInstr->getNumOperands() - 1; 6353 for (int i=0; i < numArgs; ++i) 6354 argOpers[i] = &mInstr->getOperand(i+1); 6355 6356 // x86 address has 4 operands: base, index, scale, and displacement 6357 int lastAddrIndx = 3; // [0,3] 6358 int valArgIndx = 4; 6359 6360 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6361 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6362 for (int i=0; i <= lastAddrIndx; ++i) 6363 (*MIB).addOperand(*argOpers[i]); 6364 6365 // We only support register and immediate values 6366 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6367 && "invalid operand"); 6368 6369 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6370 if (argOpers[valArgIndx]->isReg()) 6371 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6372 else 6373 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6374 (*MIB).addOperand(*argOpers[valArgIndx]); 6375 6376 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6377 MIB.addReg(t1); 6378 6379 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6380 MIB.addReg(t1); 6381 MIB.addReg(t2); 6382 6383 // Generate movc 6384 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6385 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6386 MIB.addReg(t2); 6387 MIB.addReg(t1); 6388 6389 // Cmp and exchange if none has modified the memory location 6390 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6391 for (int i=0; i <= lastAddrIndx; ++i) 6392 (*MIB).addOperand(*argOpers[i]); 6393 MIB.addReg(t3); 6394 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6395 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 6396 6397 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6398 MIB.addReg(X86::EAX); 6399 6400 // insert branch 6401 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6402 6403 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 6404 return nextMBB; 6405} 6406 6407 6408MachineBasicBlock * 6409X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6410 MachineBasicBlock *BB) { 6411 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6412 switch (MI->getOpcode()) { 6413 default: assert(false && "Unexpected instr type to insert"); 6414 case X86::CMOV_FR32: 6415 case X86::CMOV_FR64: 6416 case X86::CMOV_V4F32: 6417 case X86::CMOV_V2F64: 6418 case X86::CMOV_V2I64: { 6419 // To "insert" a SELECT_CC instruction, we actually have to insert the 6420 // diamond control-flow pattern. The incoming instruction knows the 6421 // destination vreg to set, the condition code register to branch on, the 6422 // true/false values to select between, and a branch opcode to use. 6423 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6424 MachineFunction::iterator It = BB; 6425 ++It; 6426 6427 // thisMBB: 6428 // ... 6429 // TrueVal = ... 6430 // cmpTY ccX, r1, r2 6431 // bCC copy1MBB 6432 // fallthrough --> copy0MBB 6433 MachineBasicBlock *thisMBB = BB; 6434 MachineFunction *F = BB->getParent(); 6435 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6436 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6437 unsigned Opc = 6438 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6439 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6440 F->insert(It, copy0MBB); 6441 F->insert(It, sinkMBB); 6442 // Update machine-CFG edges by transferring all successors of the current 6443 // block to the new block which will contain the Phi node for the select. 6444 sinkMBB->transferSuccessors(BB); 6445 6446 // Add the true and fallthrough blocks as its successors. 6447 BB->addSuccessor(copy0MBB); 6448 BB->addSuccessor(sinkMBB); 6449 6450 // copy0MBB: 6451 // %FalseValue = ... 6452 // # fallthrough to sinkMBB 6453 BB = copy0MBB; 6454 6455 // Update machine-CFG edges 6456 BB->addSuccessor(sinkMBB); 6457 6458 // sinkMBB: 6459 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6460 // ... 6461 BB = sinkMBB; 6462 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6463 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6464 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6465 6466 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6467 return BB; 6468 } 6469 6470 case X86::FP32_TO_INT16_IN_MEM: 6471 case X86::FP32_TO_INT32_IN_MEM: 6472 case X86::FP32_TO_INT64_IN_MEM: 6473 case X86::FP64_TO_INT16_IN_MEM: 6474 case X86::FP64_TO_INT32_IN_MEM: 6475 case X86::FP64_TO_INT64_IN_MEM: 6476 case X86::FP80_TO_INT16_IN_MEM: 6477 case X86::FP80_TO_INT32_IN_MEM: 6478 case X86::FP80_TO_INT64_IN_MEM: { 6479 // Change the floating point control register to use "round towards zero" 6480 // mode when truncating to an integer value. 6481 MachineFunction *F = BB->getParent(); 6482 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6483 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6484 6485 // Load the old value of the high byte of the control word... 6486 unsigned OldCW = 6487 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6488 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6489 6490 // Set the high part to be round to zero... 6491 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6492 .addImm(0xC7F); 6493 6494 // Reload the modified control word now... 6495 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6496 6497 // Restore the memory image of control word to original value 6498 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6499 .addReg(OldCW); 6500 6501 // Get the X86 opcode to use. 6502 unsigned Opc; 6503 switch (MI->getOpcode()) { 6504 default: assert(0 && "illegal opcode!"); 6505 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6506 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6507 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6508 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6509 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6510 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6511 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6512 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6513 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6514 } 6515 6516 X86AddressMode AM; 6517 MachineOperand &Op = MI->getOperand(0); 6518 if (Op.isRegister()) { 6519 AM.BaseType = X86AddressMode::RegBase; 6520 AM.Base.Reg = Op.getReg(); 6521 } else { 6522 AM.BaseType = X86AddressMode::FrameIndexBase; 6523 AM.Base.FrameIndex = Op.getIndex(); 6524 } 6525 Op = MI->getOperand(1); 6526 if (Op.isImmediate()) 6527 AM.Scale = Op.getImm(); 6528 Op = MI->getOperand(2); 6529 if (Op.isImmediate()) 6530 AM.IndexReg = Op.getImm(); 6531 Op = MI->getOperand(3); 6532 if (Op.isGlobalAddress()) { 6533 AM.GV = Op.getGlobal(); 6534 } else { 6535 AM.Disp = Op.getImm(); 6536 } 6537 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6538 .addReg(MI->getOperand(4).getReg()); 6539 6540 // Reload the original control word now. 6541 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6542 6543 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6544 return BB; 6545 } 6546 case X86::ATOMAND32: 6547 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6548 X86::AND32ri, X86::MOV32rm, 6549 X86::LCMPXCHG32, X86::MOV32rr, 6550 X86::NOT32r, X86::EAX, 6551 X86::GR32RegisterClass); 6552 case X86::ATOMOR32: 6553 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6554 X86::OR32ri, X86::MOV32rm, 6555 X86::LCMPXCHG32, X86::MOV32rr, 6556 X86::NOT32r, X86::EAX, 6557 X86::GR32RegisterClass); 6558 case X86::ATOMXOR32: 6559 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6560 X86::XOR32ri, X86::MOV32rm, 6561 X86::LCMPXCHG32, X86::MOV32rr, 6562 X86::NOT32r, X86::EAX, 6563 X86::GR32RegisterClass); 6564 case X86::ATOMNAND32: 6565 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6566 X86::AND32ri, X86::MOV32rm, 6567 X86::LCMPXCHG32, X86::MOV32rr, 6568 X86::NOT32r, X86::EAX, 6569 X86::GR32RegisterClass, true); 6570 case X86::ATOMMIN32: 6571 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6572 case X86::ATOMMAX32: 6573 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6574 case X86::ATOMUMIN32: 6575 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6576 case X86::ATOMUMAX32: 6577 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6578 6579 case X86::ATOMAND16: 6580 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6581 X86::AND16ri, X86::MOV16rm, 6582 X86::LCMPXCHG16, X86::MOV16rr, 6583 X86::NOT16r, X86::AX, 6584 X86::GR16RegisterClass); 6585 case X86::ATOMOR16: 6586 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 6587 X86::OR16ri, X86::MOV16rm, 6588 X86::LCMPXCHG16, X86::MOV16rr, 6589 X86::NOT16r, X86::AX, 6590 X86::GR16RegisterClass); 6591 case X86::ATOMXOR16: 6592 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 6593 X86::XOR16ri, X86::MOV16rm, 6594 X86::LCMPXCHG16, X86::MOV16rr, 6595 X86::NOT16r, X86::AX, 6596 X86::GR16RegisterClass); 6597 case X86::ATOMNAND16: 6598 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6599 X86::AND16ri, X86::MOV16rm, 6600 X86::LCMPXCHG16, X86::MOV16rr, 6601 X86::NOT16r, X86::AX, 6602 X86::GR16RegisterClass, true); 6603 case X86::ATOMMIN16: 6604 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 6605 case X86::ATOMMAX16: 6606 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 6607 case X86::ATOMUMIN16: 6608 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 6609 case X86::ATOMUMAX16: 6610 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 6611 6612 case X86::ATOMAND8: 6613 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6614 X86::AND8ri, X86::MOV8rm, 6615 X86::LCMPXCHG8, X86::MOV8rr, 6616 X86::NOT8r, X86::AL, 6617 X86::GR8RegisterClass); 6618 case X86::ATOMOR8: 6619 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 6620 X86::OR8ri, X86::MOV8rm, 6621 X86::LCMPXCHG8, X86::MOV8rr, 6622 X86::NOT8r, X86::AL, 6623 X86::GR8RegisterClass); 6624 case X86::ATOMXOR8: 6625 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 6626 X86::XOR8ri, X86::MOV8rm, 6627 X86::LCMPXCHG8, X86::MOV8rr, 6628 X86::NOT8r, X86::AL, 6629 X86::GR8RegisterClass); 6630 case X86::ATOMNAND8: 6631 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6632 X86::AND8ri, X86::MOV8rm, 6633 X86::LCMPXCHG8, X86::MOV8rr, 6634 X86::NOT8r, X86::AL, 6635 X86::GR8RegisterClass, true); 6636 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 6637 case X86::ATOMAND64: 6638 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 6639 X86::AND64ri32, X86::MOV64rm, 6640 X86::LCMPXCHG64, X86::MOV64rr, 6641 X86::NOT64r, X86::RAX, 6642 X86::GR64RegisterClass); 6643 case X86::ATOMOR64: 6644 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, 6645 X86::OR64ri32, X86::MOV64rm, 6646 X86::LCMPXCHG64, X86::MOV64rr, 6647 X86::NOT64r, X86::RAX, 6648 X86::GR64RegisterClass); 6649 case X86::ATOMXOR64: 6650 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, 6651 X86::XOR64ri32, X86::MOV64rm, 6652 X86::LCMPXCHG64, X86::MOV64rr, 6653 X86::NOT64r, X86::RAX, 6654 X86::GR64RegisterClass); 6655 case X86::ATOMNAND64: 6656 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 6657 X86::AND64ri32, X86::MOV64rm, 6658 X86::LCMPXCHG64, X86::MOV64rr, 6659 X86::NOT64r, X86::RAX, 6660 X86::GR64RegisterClass, true); 6661 case X86::ATOMMIN64: 6662 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); 6663 case X86::ATOMMAX64: 6664 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); 6665 case X86::ATOMUMIN64: 6666 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); 6667 case X86::ATOMUMAX64: 6668 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); 6669 } 6670} 6671 6672//===----------------------------------------------------------------------===// 6673// X86 Optimization Hooks 6674//===----------------------------------------------------------------------===// 6675 6676void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 6677 const APInt &Mask, 6678 APInt &KnownZero, 6679 APInt &KnownOne, 6680 const SelectionDAG &DAG, 6681 unsigned Depth) const { 6682 unsigned Opc = Op.getOpcode(); 6683 assert((Opc >= ISD::BUILTIN_OP_END || 6684 Opc == ISD::INTRINSIC_WO_CHAIN || 6685 Opc == ISD::INTRINSIC_W_CHAIN || 6686 Opc == ISD::INTRINSIC_VOID) && 6687 "Should use MaskedValueIsZero if you don't know whether Op" 6688 " is a target node!"); 6689 6690 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6691 switch (Opc) { 6692 default: break; 6693 case X86ISD::SETCC: 6694 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6695 Mask.getBitWidth() - 1); 6696 break; 6697 } 6698} 6699 6700/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6701/// node is a GlobalAddress + offset. 6702bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6703 GlobalValue* &GA, int64_t &Offset) const{ 6704 if (N->getOpcode() == X86ISD::Wrapper) { 6705 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6706 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6707 return true; 6708 } 6709 } 6710 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6711} 6712 6713static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6714 const TargetLowering &TLI) { 6715 GlobalValue *GV; 6716 int64_t Offset = 0; 6717 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6718 return (GV->getAlignment() >= N && (Offset % N) == 0); 6719 // DAG combine handles the stack object case. 6720 return false; 6721} 6722 6723static bool EltsFromConsecutiveLoads(SDNode *N, SDValue PermMask, 6724 unsigned NumElems, MVT EVT, 6725 SDNode *&Base, 6726 SelectionDAG &DAG, MachineFrameInfo *MFI, 6727 const TargetLowering &TLI) { 6728 Base = NULL; 6729 for (unsigned i = 0; i < NumElems; ++i) { 6730 SDValue Idx = PermMask.getOperand(i); 6731 if (Idx.getOpcode() == ISD::UNDEF) { 6732 if (!Base) 6733 return false; 6734 continue; 6735 } 6736 6737 SDValue Elt = DAG.getShuffleScalarElt(N, i); 6738 if (!Elt.getNode() || 6739 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) 6740 return false; 6741 if (!Base) { 6742 Base = Elt.getNode(); 6743 if (Base->getOpcode() == ISD::UNDEF) 6744 return false; 6745 continue; 6746 } 6747 if (Elt.getOpcode() == ISD::UNDEF) 6748 continue; 6749 6750 if (!TLI.isConsecutiveLoad(Elt.getNode(), Base, 6751 EVT.getSizeInBits()/8, i, MFI)) 6752 return false; 6753 } 6754 return true; 6755} 6756 6757/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6758/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6759/// if the load addresses are consecutive, non-overlapping, and in the right 6760/// order. 6761static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6762 const TargetLowering &TLI) { 6763 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6764 MVT VT = N->getValueType(0); 6765 MVT EVT = VT.getVectorElementType(); 6766 SDValue PermMask = N->getOperand(2); 6767 unsigned NumElems = PermMask.getNumOperands(); 6768 SDNode *Base = NULL; 6769 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6770 DAG, MFI, TLI)) 6771 return SDValue(); 6772 6773 LoadSDNode *LD = cast<LoadSDNode>(Base); 6774 if (isBaseAlignmentOfN(16, Base->getOperand(1).getNode(), TLI)) 6775 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6776 LD->getSrcValueOffset(), LD->isVolatile()); 6777 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6778 LD->getSrcValueOffset(), LD->isVolatile(), 6779 LD->getAlignment()); 6780} 6781 6782/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6783static SDValue PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6784 const X86Subtarget *Subtarget, 6785 const TargetLowering &TLI) { 6786 unsigned NumOps = N->getNumOperands(); 6787 6788 // Ignore single operand BUILD_VECTOR. 6789 if (NumOps == 1) 6790 return SDValue(); 6791 6792 MVT VT = N->getValueType(0); 6793 MVT EVT = VT.getVectorElementType(); 6794 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6795 // We are looking for load i64 and zero extend. We want to transform 6796 // it before legalizer has a chance to expand it. Also look for i64 6797 // BUILD_PAIR bit casted to f64. 6798 return SDValue(); 6799 // This must be an insertion into a zero vector. 6800 SDValue HighElt = N->getOperand(1); 6801 if (!isZeroNode(HighElt)) 6802 return SDValue(); 6803 6804 // Value must be a load. 6805 SDNode *Base = N->getOperand(0).getNode(); 6806 if (!isa<LoadSDNode>(Base)) { 6807 if (Base->getOpcode() != ISD::BIT_CONVERT) 6808 return SDValue(); 6809 Base = Base->getOperand(0).getNode(); 6810 if (!isa<LoadSDNode>(Base)) 6811 return SDValue(); 6812 } 6813 6814 // Transform it into VZEXT_LOAD addr. 6815 LoadSDNode *LD = cast<LoadSDNode>(Base); 6816 6817 // Load must not be an extload. 6818 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6819 return SDValue(); 6820 6821 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6822} 6823 6824/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6825static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6826 const X86Subtarget *Subtarget) { 6827 SDValue Cond = N->getOperand(0); 6828 6829 // If we have SSE[12] support, try to form min/max nodes. 6830 if (Subtarget->hasSSE2() && 6831 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6832 if (Cond.getOpcode() == ISD::SETCC) { 6833 // Get the LHS/RHS of the select. 6834 SDValue LHS = N->getOperand(1); 6835 SDValue RHS = N->getOperand(2); 6836 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6837 6838 unsigned Opcode = 0; 6839 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6840 switch (CC) { 6841 default: break; 6842 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6843 case ISD::SETULE: 6844 case ISD::SETLE: 6845 if (!UnsafeFPMath) break; 6846 // FALL THROUGH. 6847 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6848 case ISD::SETLT: 6849 Opcode = X86ISD::FMIN; 6850 break; 6851 6852 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6853 case ISD::SETUGT: 6854 case ISD::SETGT: 6855 if (!UnsafeFPMath) break; 6856 // FALL THROUGH. 6857 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6858 case ISD::SETGE: 6859 Opcode = X86ISD::FMAX; 6860 break; 6861 } 6862 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6863 switch (CC) { 6864 default: break; 6865 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6866 case ISD::SETUGT: 6867 case ISD::SETGT: 6868 if (!UnsafeFPMath) break; 6869 // FALL THROUGH. 6870 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6871 case ISD::SETGE: 6872 Opcode = X86ISD::FMIN; 6873 break; 6874 6875 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6876 case ISD::SETULE: 6877 case ISD::SETLE: 6878 if (!UnsafeFPMath) break; 6879 // FALL THROUGH. 6880 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6881 case ISD::SETLT: 6882 Opcode = X86ISD::FMAX; 6883 break; 6884 } 6885 } 6886 6887 if (Opcode) 6888 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6889 } 6890 6891 } 6892 6893 return SDValue(); 6894} 6895 6896/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6897static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6898 const X86Subtarget *Subtarget) { 6899 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6900 // the FP state in cases where an emms may be missing. 6901 // A preferable solution to the general problem is to figure out the right 6902 // places to insert EMMS. This qualifies as a quick hack. 6903 StoreSDNode *St = cast<StoreSDNode>(N); 6904 if (St->getValue().getValueType().isVector() && 6905 St->getValue().getValueType().getSizeInBits() == 64 && 6906 isa<LoadSDNode>(St->getValue()) && 6907 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6908 St->getChain().hasOneUse() && !St->isVolatile()) { 6909 SDNode* LdVal = St->getValue().getNode(); 6910 LoadSDNode *Ld = 0; 6911 int TokenFactorIndex = -1; 6912 SmallVector<SDValue, 8> Ops; 6913 SDNode* ChainVal = St->getChain().getNode(); 6914 // Must be a store of a load. We currently handle two cases: the load 6915 // is a direct child, and it's under an intervening TokenFactor. It is 6916 // possible to dig deeper under nested TokenFactors. 6917 if (ChainVal == LdVal) 6918 Ld = cast<LoadSDNode>(St->getChain()); 6919 else if (St->getValue().hasOneUse() && 6920 ChainVal->getOpcode() == ISD::TokenFactor) { 6921 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6922 if (ChainVal->getOperand(i).getNode() == LdVal) { 6923 TokenFactorIndex = i; 6924 Ld = cast<LoadSDNode>(St->getValue()); 6925 } else 6926 Ops.push_back(ChainVal->getOperand(i)); 6927 } 6928 } 6929 if (Ld) { 6930 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6931 if (Subtarget->is64Bit()) { 6932 SDValue NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6933 Ld->getBasePtr(), Ld->getSrcValue(), 6934 Ld->getSrcValueOffset(), Ld->isVolatile(), 6935 Ld->getAlignment()); 6936 SDValue NewChain = NewLd.getValue(1); 6937 if (TokenFactorIndex != -1) { 6938 Ops.push_back(NewChain); 6939 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6940 Ops.size()); 6941 } 6942 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6943 St->getSrcValue(), St->getSrcValueOffset(), 6944 St->isVolatile(), St->getAlignment()); 6945 } 6946 6947 // Otherwise, lower to two 32-bit copies. 6948 SDValue LoAddr = Ld->getBasePtr(); 6949 SDValue HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6950 DAG.getConstant(4, MVT::i32)); 6951 6952 SDValue LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6953 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6954 Ld->isVolatile(), Ld->getAlignment()); 6955 SDValue HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6956 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6957 Ld->isVolatile(), 6958 MinAlign(Ld->getAlignment(), 4)); 6959 6960 SDValue NewChain = LoLd.getValue(1); 6961 if (TokenFactorIndex != -1) { 6962 Ops.push_back(LoLd); 6963 Ops.push_back(HiLd); 6964 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6965 Ops.size()); 6966 } 6967 6968 LoAddr = St->getBasePtr(); 6969 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6970 DAG.getConstant(4, MVT::i32)); 6971 6972 SDValue LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6973 St->getSrcValue(), St->getSrcValueOffset(), 6974 St->isVolatile(), St->getAlignment()); 6975 SDValue HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6976 St->getSrcValue(), 6977 St->getSrcValueOffset() + 4, 6978 St->isVolatile(), 6979 MinAlign(St->getAlignment(), 4)); 6980 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6981 } 6982 } 6983 return SDValue(); 6984} 6985 6986/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6987/// X86ISD::FXOR nodes. 6988static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6989 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6990 // F[X]OR(0.0, x) -> x 6991 // F[X]OR(x, 0.0) -> x 6992 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6993 if (C->getValueAPF().isPosZero()) 6994 return N->getOperand(1); 6995 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6996 if (C->getValueAPF().isPosZero()) 6997 return N->getOperand(0); 6998 return SDValue(); 6999} 7000 7001/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 7002static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 7003 // FAND(0.0, x) -> 0.0 7004 // FAND(x, 0.0) -> 0.0 7005 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 7006 if (C->getValueAPF().isPosZero()) 7007 return N->getOperand(0); 7008 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 7009 if (C->getValueAPF().isPosZero()) 7010 return N->getOperand(1); 7011 return SDValue(); 7012} 7013 7014 7015SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 7016 DAGCombinerInfo &DCI) const { 7017 SelectionDAG &DAG = DCI.DAG; 7018 switch (N->getOpcode()) { 7019 default: break; 7020 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 7021 case ISD::BUILD_VECTOR: 7022 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 7023 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 7024 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 7025 case X86ISD::FXOR: 7026 case X86ISD::FOR: return PerformFORCombine(N, DAG); 7027 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 7028 } 7029 7030 return SDValue(); 7031} 7032 7033//===----------------------------------------------------------------------===// 7034// X86 Inline Assembly Support 7035//===----------------------------------------------------------------------===// 7036 7037/// getConstraintType - Given a constraint letter, return the type of 7038/// constraint it is for this target. 7039X86TargetLowering::ConstraintType 7040X86TargetLowering::getConstraintType(const std::string &Constraint) const { 7041 if (Constraint.size() == 1) { 7042 switch (Constraint[0]) { 7043 case 'A': 7044 case 'f': 7045 case 'r': 7046 case 'R': 7047 case 'l': 7048 case 'q': 7049 case 'Q': 7050 case 'x': 7051 case 'y': 7052 case 'Y': 7053 return C_RegisterClass; 7054 default: 7055 break; 7056 } 7057 } 7058 return TargetLowering::getConstraintType(Constraint); 7059} 7060 7061/// LowerXConstraint - try to replace an X constraint, which matches anything, 7062/// with another that has more specific requirements based on the type of the 7063/// corresponding operand. 7064const char *X86TargetLowering:: 7065LowerXConstraint(MVT ConstraintVT) const { 7066 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 7067 // 'f' like normal targets. 7068 if (ConstraintVT.isFloatingPoint()) { 7069 if (Subtarget->hasSSE2()) 7070 return "Y"; 7071 if (Subtarget->hasSSE1()) 7072 return "x"; 7073 } 7074 7075 return TargetLowering::LowerXConstraint(ConstraintVT); 7076} 7077 7078/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 7079/// vector. If it is invalid, don't add anything to Ops. 7080void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 7081 char Constraint, 7082 std::vector<SDValue>&Ops, 7083 SelectionDAG &DAG) const { 7084 SDValue Result(0, 0); 7085 7086 switch (Constraint) { 7087 default: break; 7088 case 'I': 7089 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 7090 if (C->getValue() <= 31) { 7091 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 7092 break; 7093 } 7094 } 7095 return; 7096 case 'N': 7097 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 7098 if (C->getValue() <= 255) { 7099 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 7100 break; 7101 } 7102 } 7103 return; 7104 case 'i': { 7105 // Literal immediates are always ok. 7106 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 7107 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 7108 break; 7109 } 7110 7111 // If we are in non-pic codegen mode, we allow the address of a global (with 7112 // an optional displacement) to be used with 'i'. 7113 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 7114 int64_t Offset = 0; 7115 7116 // Match either (GA) or (GA+C) 7117 if (GA) { 7118 Offset = GA->getOffset(); 7119 } else if (Op.getOpcode() == ISD::ADD) { 7120 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7121 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7122 if (C && GA) { 7123 Offset = GA->getOffset()+C->getValue(); 7124 } else { 7125 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7126 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7127 if (C && GA) 7128 Offset = GA->getOffset()+C->getValue(); 7129 else 7130 C = 0, GA = 0; 7131 } 7132 } 7133 7134 if (GA) { 7135 // If addressing this global requires a load (e.g. in PIC mode), we can't 7136 // match. 7137 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 7138 false)) 7139 return; 7140 7141 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 7142 Offset); 7143 Result = Op; 7144 break; 7145 } 7146 7147 // Otherwise, not valid for this mode. 7148 return; 7149 } 7150 } 7151 7152 if (Result.getNode()) { 7153 Ops.push_back(Result); 7154 return; 7155 } 7156 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 7157} 7158 7159std::vector<unsigned> X86TargetLowering:: 7160getRegClassForInlineAsmConstraint(const std::string &Constraint, 7161 MVT VT) const { 7162 if (Constraint.size() == 1) { 7163 // FIXME: not handling fp-stack yet! 7164 switch (Constraint[0]) { // GCC X86 Constraint Letters 7165 default: break; // Unknown constraint letter 7166 case 'A': // EAX/EDX 7167 if (VT == MVT::i32 || VT == MVT::i64) 7168 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 7169 break; 7170 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 7171 case 'Q': // Q_REGS 7172 if (VT == MVT::i32) 7173 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 7174 else if (VT == MVT::i16) 7175 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 7176 else if (VT == MVT::i8) 7177 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 7178 else if (VT == MVT::i64) 7179 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 7180 break; 7181 } 7182 } 7183 7184 return std::vector<unsigned>(); 7185} 7186 7187std::pair<unsigned, const TargetRegisterClass*> 7188X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 7189 MVT VT) const { 7190 // First, see if this is a constraint that directly corresponds to an LLVM 7191 // register class. 7192 if (Constraint.size() == 1) { 7193 // GCC Constraint Letters 7194 switch (Constraint[0]) { 7195 default: break; 7196 case 'r': // GENERAL_REGS 7197 case 'R': // LEGACY_REGS 7198 case 'l': // INDEX_REGS 7199 if (VT == MVT::i64 && Subtarget->is64Bit()) 7200 return std::make_pair(0U, X86::GR64RegisterClass); 7201 if (VT == MVT::i32) 7202 return std::make_pair(0U, X86::GR32RegisterClass); 7203 else if (VT == MVT::i16) 7204 return std::make_pair(0U, X86::GR16RegisterClass); 7205 else if (VT == MVT::i8) 7206 return std::make_pair(0U, X86::GR8RegisterClass); 7207 break; 7208 case 'f': // FP Stack registers. 7209 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 7210 // value to the correct fpstack register class. 7211 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 7212 return std::make_pair(0U, X86::RFP32RegisterClass); 7213 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 7214 return std::make_pair(0U, X86::RFP64RegisterClass); 7215 return std::make_pair(0U, X86::RFP80RegisterClass); 7216 case 'y': // MMX_REGS if MMX allowed. 7217 if (!Subtarget->hasMMX()) break; 7218 return std::make_pair(0U, X86::VR64RegisterClass); 7219 break; 7220 case 'Y': // SSE_REGS if SSE2 allowed 7221 if (!Subtarget->hasSSE2()) break; 7222 // FALL THROUGH. 7223 case 'x': // SSE_REGS if SSE1 allowed 7224 if (!Subtarget->hasSSE1()) break; 7225 7226 switch (VT.getSimpleVT()) { 7227 default: break; 7228 // Scalar SSE types. 7229 case MVT::f32: 7230 case MVT::i32: 7231 return std::make_pair(0U, X86::FR32RegisterClass); 7232 case MVT::f64: 7233 case MVT::i64: 7234 return std::make_pair(0U, X86::FR64RegisterClass); 7235 // Vector types. 7236 case MVT::v16i8: 7237 case MVT::v8i16: 7238 case MVT::v4i32: 7239 case MVT::v2i64: 7240 case MVT::v4f32: 7241 case MVT::v2f64: 7242 return std::make_pair(0U, X86::VR128RegisterClass); 7243 } 7244 break; 7245 } 7246 } 7247 7248 // Use the default implementation in TargetLowering to convert the register 7249 // constraint into a member of a register class. 7250 std::pair<unsigned, const TargetRegisterClass*> Res; 7251 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 7252 7253 // Not found as a standard register? 7254 if (Res.second == 0) { 7255 // GCC calls "st(0)" just plain "st". 7256 if (StringsEqualNoCase("{st}", Constraint)) { 7257 Res.first = X86::ST0; 7258 Res.second = X86::RFP80RegisterClass; 7259 } 7260 7261 return Res; 7262 } 7263 7264 // Otherwise, check to see if this is a register class of the wrong value 7265 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 7266 // turn into {ax},{dx}. 7267 if (Res.second->hasType(VT)) 7268 return Res; // Correct type already, nothing to do. 7269 7270 // All of the single-register GCC register classes map their values onto 7271 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 7272 // really want an 8-bit or 32-bit register, map to the appropriate register 7273 // class and return the appropriate register. 7274 if (Res.second == X86::GR16RegisterClass) { 7275 if (VT == MVT::i8) { 7276 unsigned DestReg = 0; 7277 switch (Res.first) { 7278 default: break; 7279 case X86::AX: DestReg = X86::AL; break; 7280 case X86::DX: DestReg = X86::DL; break; 7281 case X86::CX: DestReg = X86::CL; break; 7282 case X86::BX: DestReg = X86::BL; break; 7283 } 7284 if (DestReg) { 7285 Res.first = DestReg; 7286 Res.second = Res.second = X86::GR8RegisterClass; 7287 } 7288 } else if (VT == MVT::i32) { 7289 unsigned DestReg = 0; 7290 switch (Res.first) { 7291 default: break; 7292 case X86::AX: DestReg = X86::EAX; break; 7293 case X86::DX: DestReg = X86::EDX; break; 7294 case X86::CX: DestReg = X86::ECX; break; 7295 case X86::BX: DestReg = X86::EBX; break; 7296 case X86::SI: DestReg = X86::ESI; break; 7297 case X86::DI: DestReg = X86::EDI; break; 7298 case X86::BP: DestReg = X86::EBP; break; 7299 case X86::SP: DestReg = X86::ESP; break; 7300 } 7301 if (DestReg) { 7302 Res.first = DestReg; 7303 Res.second = Res.second = X86::GR32RegisterClass; 7304 } 7305 } else if (VT == MVT::i64) { 7306 unsigned DestReg = 0; 7307 switch (Res.first) { 7308 default: break; 7309 case X86::AX: DestReg = X86::RAX; break; 7310 case X86::DX: DestReg = X86::RDX; break; 7311 case X86::CX: DestReg = X86::RCX; break; 7312 case X86::BX: DestReg = X86::RBX; break; 7313 case X86::SI: DestReg = X86::RSI; break; 7314 case X86::DI: DestReg = X86::RDI; break; 7315 case X86::BP: DestReg = X86::RBP; break; 7316 case X86::SP: DestReg = X86::RSP; break; 7317 } 7318 if (DestReg) { 7319 Res.first = DestReg; 7320 Res.second = Res.second = X86::GR64RegisterClass; 7321 } 7322 } 7323 } else if (Res.second == X86::FR32RegisterClass || 7324 Res.second == X86::FR64RegisterClass || 7325 Res.second == X86::VR128RegisterClass) { 7326 // Handle references to XMM physical registers that got mapped into the 7327 // wrong class. This can happen with constraints like {xmm0} where the 7328 // target independent register mapper will just pick the first match it can 7329 // find, ignoring the required type. 7330 if (VT == MVT::f32) 7331 Res.second = X86::FR32RegisterClass; 7332 else if (VT == MVT::f64) 7333 Res.second = X86::FR64RegisterClass; 7334 else if (X86::VR128RegisterClass->hasType(VT)) 7335 Res.second = X86::VR128RegisterClass; 7336 } 7337 7338 return Res; 7339} 7340