X86ISelLowering.cpp revision 558cc5a914118bba0e7843f1878f6137fea88d91
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "x86-isel" 16#include "X86.h" 17#include "X86InstrBuilder.h" 18#include "X86ISelLowering.h" 19#include "X86TargetMachine.h" 20#include "X86TargetObjectFile.h" 21#include "Utils/X86ShuffleDecode.h" 22#include "llvm/CallingConv.h" 23#include "llvm/Constants.h" 24#include "llvm/DerivedTypes.h" 25#include "llvm/GlobalAlias.h" 26#include "llvm/GlobalVariable.h" 27#include "llvm/Function.h" 28#include "llvm/Instructions.h" 29#include "llvm/Intrinsics.h" 30#include "llvm/LLVMContext.h" 31#include "llvm/CodeGen/IntrinsicLowering.h" 32#include "llvm/CodeGen/MachineFrameInfo.h" 33#include "llvm/CodeGen/MachineFunction.h" 34#include "llvm/CodeGen/MachineInstrBuilder.h" 35#include "llvm/CodeGen/MachineJumpTableInfo.h" 36#include "llvm/CodeGen/MachineModuleInfo.h" 37#include "llvm/CodeGen/MachineRegisterInfo.h" 38#include "llvm/CodeGen/PseudoSourceValue.h" 39#include "llvm/MC/MCAsmInfo.h" 40#include "llvm/MC/MCContext.h" 41#include "llvm/MC/MCExpr.h" 42#include "llvm/MC/MCSymbol.h" 43#include "llvm/ADT/BitVector.h" 44#include "llvm/ADT/SmallSet.h" 45#include "llvm/ADT/Statistic.h" 46#include "llvm/ADT/StringExtras.h" 47#include "llvm/ADT/VectorExtras.h" 48#include "llvm/Support/CallSite.h" 49#include "llvm/Support/Debug.h" 50#include "llvm/Support/Dwarf.h" 51#include "llvm/Support/ErrorHandling.h" 52#include "llvm/Support/MathExtras.h" 53#include "llvm/Support/raw_ostream.h" 54using namespace llvm; 55using namespace dwarf; 56 57STATISTIC(NumTailCalls, "Number of tail calls"); 58 59// Forward declarations. 60static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 61 SDValue V2); 62 63static SDValue Insert128BitVector(SDValue Result, 64 SDValue Vec, 65 SDValue Idx, 66 SelectionDAG &DAG, 67 DebugLoc dl); 68 69static SDValue Extract128BitVector(SDValue Vec, 70 SDValue Idx, 71 SelectionDAG &DAG, 72 DebugLoc dl); 73 74/// Generate a DAG to grab 128-bits from a vector > 128 bits. This 75/// sets things up to match to an AVX VEXTRACTF128 instruction or a 76/// simple subregister reference. Idx is an index in the 128 bits we 77/// want. It need not be aligned to a 128-bit bounday. That makes 78/// lowering EXTRACT_VECTOR_ELT operations easier. 79static SDValue Extract128BitVector(SDValue Vec, 80 SDValue Idx, 81 SelectionDAG &DAG, 82 DebugLoc dl) { 83 EVT VT = Vec.getValueType(); 84 assert(VT.getSizeInBits() == 256 && "Unexpected vector size!"); 85 EVT ElVT = VT.getVectorElementType(); 86 int Factor = VT.getSizeInBits()/128; 87 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, 88 VT.getVectorNumElements()/Factor); 89 90 // Extract from UNDEF is UNDEF. 91 if (Vec.getOpcode() == ISD::UNDEF) 92 return DAG.getNode(ISD::UNDEF, dl, ResultVT); 93 94 if (isa<ConstantSDNode>(Idx)) { 95 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 96 97 // Extract the relevant 128 bits. Generate an EXTRACT_SUBVECTOR 98 // we can match to VEXTRACTF128. 99 unsigned ElemsPerChunk = 128 / ElVT.getSizeInBits(); 100 101 // This is the index of the first element of the 128-bit chunk 102 // we want. 103 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / 128) 104 * ElemsPerChunk); 105 106 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); 107 SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, 108 VecIdx); 109 110 return Result; 111 } 112 113 return SDValue(); 114} 115 116/// Generate a DAG to put 128-bits into a vector > 128 bits. This 117/// sets things up to match to an AVX VINSERTF128 instruction or a 118/// simple superregister reference. Idx is an index in the 128 bits 119/// we want. It need not be aligned to a 128-bit bounday. That makes 120/// lowering INSERT_VECTOR_ELT operations easier. 121static SDValue Insert128BitVector(SDValue Result, 122 SDValue Vec, 123 SDValue Idx, 124 SelectionDAG &DAG, 125 DebugLoc dl) { 126 if (isa<ConstantSDNode>(Idx)) { 127 EVT VT = Vec.getValueType(); 128 assert(VT.getSizeInBits() == 128 && "Unexpected vector size!"); 129 130 EVT ElVT = VT.getVectorElementType(); 131 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 132 EVT ResultVT = Result.getValueType(); 133 134 // Insert the relevant 128 bits. 135 unsigned ElemsPerChunk = 128/ElVT.getSizeInBits(); 136 137 // This is the index of the first element of the 128-bit chunk 138 // we want. 139 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/128) 140 * ElemsPerChunk); 141 142 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); 143 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, 144 VecIdx); 145 return Result; 146 } 147 148 return SDValue(); 149} 150 151static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) { 152 const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>(); 153 bool is64Bit = Subtarget->is64Bit(); 154 155 if (Subtarget->isTargetEnvMacho()) { 156 if (is64Bit) 157 return new X8664_MachoTargetObjectFile(); 158 return new TargetLoweringObjectFileMachO(); 159 } 160 161 if (Subtarget->isTargetELF()) 162 return new TargetLoweringObjectFileELF(); 163 if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) 164 return new TargetLoweringObjectFileCOFF(); 165 llvm_unreachable("unknown subtarget type"); 166} 167 168X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 169 : TargetLowering(TM, createTLOF(TM)) { 170 Subtarget = &TM.getSubtarget<X86Subtarget>(); 171 X86ScalarSSEf64 = Subtarget->hasXMMInt(); 172 X86ScalarSSEf32 = Subtarget->hasXMM(); 173 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 174 175 RegInfo = TM.getRegisterInfo(); 176 TD = getTargetData(); 177 178 // Set up the TargetLowering object. 179 static MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }; 180 181 // X86 is weird, it always uses i8 for shift amounts and setcc results. 182 setBooleanContents(ZeroOrOneBooleanContent); 183 184 // For 64-bit since we have so many registers use the ILP scheduler, for 185 // 32-bit code use the register pressure specific scheduling. 186 if (Subtarget->is64Bit()) 187 setSchedulingPreference(Sched::ILP); 188 else 189 setSchedulingPreference(Sched::RegPressure); 190 setStackPointerRegisterToSaveRestore(X86StackPtr); 191 192 if (Subtarget->isTargetWindows() && !Subtarget->isTargetCygMing()) { 193 // Setup Windows compiler runtime calls. 194 setLibcallName(RTLIB::SDIV_I64, "_alldiv"); 195 setLibcallName(RTLIB::UDIV_I64, "_aulldiv"); 196 setLibcallName(RTLIB::SREM_I64, "_allrem"); 197 setLibcallName(RTLIB::UREM_I64, "_aullrem"); 198 setLibcallName(RTLIB::MUL_I64, "_allmul"); 199 setLibcallName(RTLIB::FPTOUINT_F64_I64, "_ftol2"); 200 setLibcallName(RTLIB::FPTOUINT_F32_I64, "_ftol2"); 201 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall); 202 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall); 203 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall); 204 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall); 205 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall); 206 setLibcallCallingConv(RTLIB::FPTOUINT_F64_I64, CallingConv::C); 207 setLibcallCallingConv(RTLIB::FPTOUINT_F32_I64, CallingConv::C); 208 } 209 210 if (Subtarget->isTargetDarwin()) { 211 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 212 setUseUnderscoreSetJmp(false); 213 setUseUnderscoreLongJmp(false); 214 } else if (Subtarget->isTargetMingw()) { 215 // MS runtime is weird: it exports _setjmp, but longjmp! 216 setUseUnderscoreSetJmp(true); 217 setUseUnderscoreLongJmp(false); 218 } else { 219 setUseUnderscoreSetJmp(true); 220 setUseUnderscoreLongJmp(true); 221 } 222 223 // Set up the register classes. 224 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 225 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 226 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 227 if (Subtarget->is64Bit()) 228 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 229 230 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 231 232 // We don't accept any truncstore of integer registers. 233 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 234 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 235 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 236 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 237 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 238 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 239 240 // SETOEQ and SETUNE require checking two conditions. 241 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); 242 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); 243 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); 244 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); 245 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); 246 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); 247 248 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 249 // operation. 250 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 251 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 252 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 253 254 if (Subtarget->is64Bit()) { 255 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 256 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 257 } else if (!UseSoftFloat) { 258 // We have an algorithm for SSE2->double, and we turn this into a 259 // 64-bit FILD followed by conditional FADD for other targets. 260 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 261 // We have an algorithm for SSE2, and we turn this into a 64-bit 262 // FILD for other targets. 263 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); 264 } 265 266 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 267 // this operation. 268 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 269 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 270 271 if (!UseSoftFloat) { 272 // SSE has no i16 to fp conversion, only i32 273 if (X86ScalarSSEf32) { 274 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 275 // f32 and f64 cases are Legal, f80 case is not 276 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 277 } else { 278 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 279 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 280 } 281 } else { 282 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 283 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); 284 } 285 286 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 287 // are Legal, f80 is custom lowered. 288 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 289 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 290 291 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 292 // this operation. 293 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 294 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 295 296 if (X86ScalarSSEf32) { 297 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 298 // f32 and f64 cases are Legal, f80 case is not 299 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 300 } else { 301 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 302 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 303 } 304 305 // Handle FP_TO_UINT by promoting the destination to a larger signed 306 // conversion. 307 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 308 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 309 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 310 311 if (Subtarget->is64Bit()) { 312 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 313 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 314 } else if (!UseSoftFloat) { 315 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 316 // Expand FP_TO_UINT into a select. 317 // FIXME: We would like to use a Custom expander here eventually to do 318 // the optimal thing for SSE vs. the default expansion in the legalizer. 319 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 320 else 321 // With SSE3 we can use fisttpll to convert to a signed i64; without 322 // SSE, we're stuck with a fistpll. 323 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); 324 } 325 326 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 327 if (!X86ScalarSSEf64) { 328 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); 329 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); 330 if (Subtarget->is64Bit()) { 331 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); 332 // Without SSE, i64->f64 goes through memory. 333 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); 334 } 335 } 336 337 // Scalar integer divide and remainder are lowered to use operations that 338 // produce two results, to match the available instructions. This exposes 339 // the two-result form to trivial CSE, which is able to combine x/y and x%y 340 // into a single instruction. 341 // 342 // Scalar integer multiply-high is also lowered to use two-result 343 // operations, to match the available instructions. However, plain multiply 344 // (low) operations are left as Legal, as there are single-result 345 // instructions for this in x86. Using the two-result multiply instructions 346 // when both high and low results are needed must be arranged by dagcombine. 347 for (unsigned i = 0, e = 4; i != e; ++i) { 348 MVT VT = IntVTs[i]; 349 setOperationAction(ISD::MULHS, VT, Expand); 350 setOperationAction(ISD::MULHU, VT, Expand); 351 setOperationAction(ISD::SDIV, VT, Expand); 352 setOperationAction(ISD::UDIV, VT, Expand); 353 setOperationAction(ISD::SREM, VT, Expand); 354 setOperationAction(ISD::UREM, VT, Expand); 355 356 // Add/Sub overflow ops with MVT::Glues are lowered to EFLAGS dependences. 357 setOperationAction(ISD::ADDC, VT, Custom); 358 setOperationAction(ISD::ADDE, VT, Custom); 359 setOperationAction(ISD::SUBC, VT, Custom); 360 setOperationAction(ISD::SUBE, VT, Custom); 361 } 362 363 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 364 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 365 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 366 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 367 if (Subtarget->is64Bit()) 368 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 369 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 370 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 371 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 372 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 373 setOperationAction(ISD::FREM , MVT::f32 , Expand); 374 setOperationAction(ISD::FREM , MVT::f64 , Expand); 375 setOperationAction(ISD::FREM , MVT::f80 , Expand); 376 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 377 378 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 379 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 380 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 381 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 382 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 383 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 384 if (Subtarget->is64Bit()) { 385 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 386 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 387 } 388 389 if (Subtarget->hasPOPCNT()) { 390 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); 391 } else { 392 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 393 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 394 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 395 if (Subtarget->is64Bit()) 396 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 397 } 398 399 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 400 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 401 402 // These should be promoted to a larger select which is supported. 403 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 404 // X86 wants to expand cmov itself. 405 setOperationAction(ISD::SELECT , MVT::i8 , Custom); 406 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 407 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 408 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 409 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 410 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 411 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 412 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 413 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 414 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 415 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 416 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 417 if (Subtarget->is64Bit()) { 418 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 419 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 420 } 421 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 422 423 // Darwin ABI issue. 424 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 425 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 426 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 427 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 428 if (Subtarget->is64Bit()) 429 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 430 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 431 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); 432 if (Subtarget->is64Bit()) { 433 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 434 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 435 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 436 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 437 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); 438 } 439 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 440 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 441 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 442 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 443 if (Subtarget->is64Bit()) { 444 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 445 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 446 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 447 } 448 449 if (Subtarget->hasXMM()) 450 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 451 452 // We may not have a libcall for MEMBARRIER so we should lower this. 453 setOperationAction(ISD::MEMBARRIER , MVT::Other, Custom); 454 455 // On X86 and X86-64, atomic operations are lowered to locked instructions. 456 // Locked instructions, in turn, have implicit fence semantics (all memory 457 // operations are flushed before issuing the locked instruction, and they 458 // are not buffered), so we can fold away the common pattern of 459 // fence-atomic-fence. 460 setShouldFoldAtomicFences(true); 461 462 // Expand certain atomics 463 for (unsigned i = 0, e = 4; i != e; ++i) { 464 MVT VT = IntVTs[i]; 465 setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom); 466 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 467 } 468 469 if (!Subtarget->is64Bit()) { 470 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom); 471 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); 472 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); 473 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom); 474 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom); 475 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom); 476 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom); 477 } 478 479 // FIXME - use subtarget debug flags 480 if (!Subtarget->isTargetDarwin() && 481 !Subtarget->isTargetELF() && 482 !Subtarget->isTargetCygMing()) { 483 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 484 } 485 486 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 487 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 488 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 489 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 490 if (Subtarget->is64Bit()) { 491 setExceptionPointerRegister(X86::RAX); 492 setExceptionSelectorRegister(X86::RDX); 493 } else { 494 setExceptionPointerRegister(X86::EAX); 495 setExceptionSelectorRegister(X86::EDX); 496 } 497 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 498 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 499 500 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 501 502 setOperationAction(ISD::TRAP, MVT::Other, Legal); 503 504 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 505 setOperationAction(ISD::VASTART , MVT::Other, Custom); 506 setOperationAction(ISD::VAEND , MVT::Other, Expand); 507 if (Subtarget->is64Bit()) { 508 setOperationAction(ISD::VAARG , MVT::Other, Custom); 509 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 510 } else { 511 setOperationAction(ISD::VAARG , MVT::Other, Expand); 512 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 513 } 514 515 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 516 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 517 setOperationAction(ISD::DYNAMIC_STACKALLOC, 518 (Subtarget->is64Bit() ? MVT::i64 : MVT::i32), 519 (Subtarget->isTargetCOFF() 520 && !Subtarget->isTargetEnvMacho() 521 ? Custom : Expand)); 522 523 if (!UseSoftFloat && X86ScalarSSEf64) { 524 // f32 and f64 use SSE. 525 // Set up the FP register classes. 526 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 527 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 528 529 // Use ANDPD to simulate FABS. 530 setOperationAction(ISD::FABS , MVT::f64, Custom); 531 setOperationAction(ISD::FABS , MVT::f32, Custom); 532 533 // Use XORP to simulate FNEG. 534 setOperationAction(ISD::FNEG , MVT::f64, Custom); 535 setOperationAction(ISD::FNEG , MVT::f32, Custom); 536 537 // Use ANDPD and ORPD to simulate FCOPYSIGN. 538 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 539 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 540 541 // Lower this to FGETSIGNx86 plus an AND. 542 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); 543 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); 544 545 // We don't support sin/cos/fmod 546 setOperationAction(ISD::FSIN , MVT::f64, Expand); 547 setOperationAction(ISD::FCOS , MVT::f64, Expand); 548 setOperationAction(ISD::FSIN , MVT::f32, Expand); 549 setOperationAction(ISD::FCOS , MVT::f32, Expand); 550 551 // Expand FP immediates into loads from the stack, except for the special 552 // cases we handle. 553 addLegalFPImmediate(APFloat(+0.0)); // xorpd 554 addLegalFPImmediate(APFloat(+0.0f)); // xorps 555 } else if (!UseSoftFloat && X86ScalarSSEf32) { 556 // Use SSE for f32, x87 for f64. 557 // Set up the FP register classes. 558 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 559 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 560 561 // Use ANDPS to simulate FABS. 562 setOperationAction(ISD::FABS , MVT::f32, Custom); 563 564 // Use XORP to simulate FNEG. 565 setOperationAction(ISD::FNEG , MVT::f32, Custom); 566 567 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 568 569 // Use ANDPS and ORPS to simulate FCOPYSIGN. 570 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 571 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 572 573 // We don't support sin/cos/fmod 574 setOperationAction(ISD::FSIN , MVT::f32, Expand); 575 setOperationAction(ISD::FCOS , MVT::f32, Expand); 576 577 // Special cases we handle for FP constants. 578 addLegalFPImmediate(APFloat(+0.0f)); // xorps 579 addLegalFPImmediate(APFloat(+0.0)); // FLD0 580 addLegalFPImmediate(APFloat(+1.0)); // FLD1 581 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 582 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 583 584 if (!UnsafeFPMath) { 585 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 586 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 587 } 588 } else if (!UseSoftFloat) { 589 // f32 and f64 in x87. 590 // Set up the FP register classes. 591 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 592 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 593 594 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 595 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 596 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 597 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 598 599 if (!UnsafeFPMath) { 600 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 601 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 602 } 603 addLegalFPImmediate(APFloat(+0.0)); // FLD0 604 addLegalFPImmediate(APFloat(+1.0)); // FLD1 605 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 606 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 607 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 608 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 609 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 610 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 611 } 612 613 // We don't support FMA. 614 setOperationAction(ISD::FMA, MVT::f64, Expand); 615 setOperationAction(ISD::FMA, MVT::f32, Expand); 616 617 // Long double always uses X87. 618 if (!UseSoftFloat) { 619 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 620 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 621 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 622 { 623 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended); 624 addLegalFPImmediate(TmpFlt); // FLD0 625 TmpFlt.changeSign(); 626 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 627 628 bool ignored; 629 APFloat TmpFlt2(+1.0); 630 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, 631 &ignored); 632 addLegalFPImmediate(TmpFlt2); // FLD1 633 TmpFlt2.changeSign(); 634 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 635 } 636 637 if (!UnsafeFPMath) { 638 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 639 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 640 } 641 642 setOperationAction(ISD::FMA, MVT::f80, Expand); 643 } 644 645 // Always use a library call for pow. 646 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 647 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 648 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 649 650 setOperationAction(ISD::FLOG, MVT::f80, Expand); 651 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 652 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 653 setOperationAction(ISD::FEXP, MVT::f80, Expand); 654 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 655 656 // First set operation action for all vector types to either promote 657 // (for widening) or expand (for scalarization). Then we will selectively 658 // turn on ones that can be effectively codegen'd. 659 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 660 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 661 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 662 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 663 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 664 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 665 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 666 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 667 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 668 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 669 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 670 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 671 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 672 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 673 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 674 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 675 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); 676 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); 677 setOperationAction(ISD::EXTRACT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand); 678 setOperationAction(ISD::INSERT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand); 679 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 680 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 681 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 682 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 683 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 684 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 685 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 686 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 687 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 688 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 689 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 690 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 691 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 692 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 693 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 694 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 695 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 696 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 697 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 698 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 699 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 700 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 701 setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand); 702 setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand); 703 setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand); 704 setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand); 705 setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand); 706 setOperationAction(ISD::FP_TO_UINT, (MVT::SimpleValueType)VT, Expand); 707 setOperationAction(ISD::FP_TO_SINT, (MVT::SimpleValueType)VT, Expand); 708 setOperationAction(ISD::UINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 709 setOperationAction(ISD::SINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 710 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,Expand); 711 setOperationAction(ISD::TRUNCATE, (MVT::SimpleValueType)VT, Expand); 712 setOperationAction(ISD::SIGN_EXTEND, (MVT::SimpleValueType)VT, Expand); 713 setOperationAction(ISD::ZERO_EXTEND, (MVT::SimpleValueType)VT, Expand); 714 setOperationAction(ISD::ANY_EXTEND, (MVT::SimpleValueType)VT, Expand); 715 for (unsigned InnerVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 716 InnerVT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++InnerVT) 717 setTruncStoreAction((MVT::SimpleValueType)VT, 718 (MVT::SimpleValueType)InnerVT, Expand); 719 setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT, Expand); 720 setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT, Expand); 721 setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT, Expand); 722 } 723 724 // FIXME: In order to prevent SSE instructions being expanded to MMX ones 725 // with -msoft-float, disable use of MMX as well. 726 if (!UseSoftFloat && Subtarget->hasMMX()) { 727 addRegisterClass(MVT::x86mmx, X86::VR64RegisterClass); 728 // No operations on x86mmx supported, everything uses intrinsics. 729 } 730 731 // MMX-sized vectors (other than x86mmx) are expected to be expanded 732 // into smaller operations. 733 setOperationAction(ISD::MULHS, MVT::v8i8, Expand); 734 setOperationAction(ISD::MULHS, MVT::v4i16, Expand); 735 setOperationAction(ISD::MULHS, MVT::v2i32, Expand); 736 setOperationAction(ISD::MULHS, MVT::v1i64, Expand); 737 setOperationAction(ISD::AND, MVT::v8i8, Expand); 738 setOperationAction(ISD::AND, MVT::v4i16, Expand); 739 setOperationAction(ISD::AND, MVT::v2i32, Expand); 740 setOperationAction(ISD::AND, MVT::v1i64, Expand); 741 setOperationAction(ISD::OR, MVT::v8i8, Expand); 742 setOperationAction(ISD::OR, MVT::v4i16, Expand); 743 setOperationAction(ISD::OR, MVT::v2i32, Expand); 744 setOperationAction(ISD::OR, MVT::v1i64, Expand); 745 setOperationAction(ISD::XOR, MVT::v8i8, Expand); 746 setOperationAction(ISD::XOR, MVT::v4i16, Expand); 747 setOperationAction(ISD::XOR, MVT::v2i32, Expand); 748 setOperationAction(ISD::XOR, MVT::v1i64, Expand); 749 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Expand); 750 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Expand); 751 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i32, Expand); 752 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Expand); 753 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); 754 setOperationAction(ISD::SELECT, MVT::v8i8, Expand); 755 setOperationAction(ISD::SELECT, MVT::v4i16, Expand); 756 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 757 setOperationAction(ISD::SELECT, MVT::v1i64, Expand); 758 setOperationAction(ISD::BITCAST, MVT::v8i8, Expand); 759 setOperationAction(ISD::BITCAST, MVT::v4i16, Expand); 760 setOperationAction(ISD::BITCAST, MVT::v2i32, Expand); 761 setOperationAction(ISD::BITCAST, MVT::v1i64, Expand); 762 763 if (!UseSoftFloat && Subtarget->hasXMM()) { 764 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 765 766 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 767 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 768 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 769 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 770 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 771 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 772 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 773 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 774 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 775 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 776 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 777 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 778 } 779 780 if (!UseSoftFloat && Subtarget->hasXMMInt()) { 781 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 782 783 // FIXME: Unfortunately -soft-float and -no-implicit-float means XMM 784 // registers cannot be used even for integer operations. 785 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 786 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 787 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 788 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 789 790 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 791 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 792 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 793 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 794 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 795 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 796 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 797 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 798 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 799 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 800 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 801 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 802 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 803 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 804 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 805 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 806 807 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 808 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 809 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 810 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 811 812 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 813 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 814 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 815 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 816 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 817 818 setOperationAction(ISD::CONCAT_VECTORS, MVT::v2f64, Custom); 819 setOperationAction(ISD::CONCAT_VECTORS, MVT::v2i64, Custom); 820 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i8, Custom); 821 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i16, Custom); 822 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 823 824 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 825 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 826 EVT VT = (MVT::SimpleValueType)i; 827 // Do not attempt to custom lower non-power-of-2 vectors 828 if (!isPowerOf2_32(VT.getVectorNumElements())) 829 continue; 830 // Do not attempt to custom lower non-128-bit vectors 831 if (!VT.is128BitVector()) 832 continue; 833 setOperationAction(ISD::BUILD_VECTOR, 834 VT.getSimpleVT().SimpleTy, Custom); 835 setOperationAction(ISD::VECTOR_SHUFFLE, 836 VT.getSimpleVT().SimpleTy, Custom); 837 setOperationAction(ISD::EXTRACT_VECTOR_ELT, 838 VT.getSimpleVT().SimpleTy, Custom); 839 } 840 841 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 842 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 843 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 844 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 845 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 846 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 847 848 if (Subtarget->is64Bit()) { 849 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 850 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 851 } 852 853 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 854 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; i++) { 855 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 856 EVT VT = SVT; 857 858 // Do not attempt to promote non-128-bit vectors 859 if (!VT.is128BitVector()) 860 continue; 861 862 setOperationAction(ISD::AND, SVT, Promote); 863 AddPromotedToType (ISD::AND, SVT, MVT::v2i64); 864 setOperationAction(ISD::OR, SVT, Promote); 865 AddPromotedToType (ISD::OR, SVT, MVT::v2i64); 866 setOperationAction(ISD::XOR, SVT, Promote); 867 AddPromotedToType (ISD::XOR, SVT, MVT::v2i64); 868 setOperationAction(ISD::LOAD, SVT, Promote); 869 AddPromotedToType (ISD::LOAD, SVT, MVT::v2i64); 870 setOperationAction(ISD::SELECT, SVT, Promote); 871 AddPromotedToType (ISD::SELECT, SVT, MVT::v2i64); 872 } 873 874 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 875 876 // Custom lower v2i64 and v2f64 selects. 877 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 878 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 879 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 880 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 881 882 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 883 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 884 } 885 886 if (Subtarget->hasSSE41()) { 887 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 888 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 889 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 890 setOperationAction(ISD::FRINT, MVT::f32, Legal); 891 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 892 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 893 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 894 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 895 setOperationAction(ISD::FRINT, MVT::f64, Legal); 896 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 897 898 // FIXME: Do we need to handle scalar-to-vector here? 899 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 900 901 // Can turn SHL into an integer multiply. 902 setOperationAction(ISD::SHL, MVT::v4i32, Custom); 903 setOperationAction(ISD::SHL, MVT::v16i8, Custom); 904 905 // i8 and i16 vectors are custom , because the source register and source 906 // source memory operand types are not the same width. f32 vectors are 907 // custom since the immediate controlling the insert encodes additional 908 // information. 909 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 910 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 911 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 912 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 913 914 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 915 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 916 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 917 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 918 919 if (Subtarget->is64Bit()) { 920 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 921 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 922 } 923 } 924 925 if (Subtarget->hasSSE2()) { 926 setOperationAction(ISD::SRL, MVT::v2i64, Custom); 927 setOperationAction(ISD::SRL, MVT::v4i32, Custom); 928 setOperationAction(ISD::SRL, MVT::v16i8, Custom); 929 930 setOperationAction(ISD::SHL, MVT::v2i64, Custom); 931 setOperationAction(ISD::SHL, MVT::v4i32, Custom); 932 setOperationAction(ISD::SHL, MVT::v8i16, Custom); 933 934 setOperationAction(ISD::SRA, MVT::v4i32, Custom); 935 setOperationAction(ISD::SRA, MVT::v8i16, Custom); 936 } 937 938 if (Subtarget->hasSSE42()) 939 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 940 941 if (!UseSoftFloat && Subtarget->hasAVX()) { 942 addRegisterClass(MVT::v32i8, X86::VR256RegisterClass); 943 addRegisterClass(MVT::v16i16, X86::VR256RegisterClass); 944 addRegisterClass(MVT::v8i32, X86::VR256RegisterClass); 945 addRegisterClass(MVT::v8f32, X86::VR256RegisterClass); 946 addRegisterClass(MVT::v4i64, X86::VR256RegisterClass); 947 addRegisterClass(MVT::v4f64, X86::VR256RegisterClass); 948 949 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); 950 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); 951 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); 952 953 setOperationAction(ISD::FADD, MVT::v8f32, Legal); 954 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); 955 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); 956 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); 957 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); 958 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); 959 960 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 961 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 962 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 963 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 964 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 965 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); 966 967 // Custom lower several nodes for 256-bit types. 968 for (unsigned i = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 969 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { 970 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 971 EVT VT = SVT; 972 973 // Extract subvector is special because the value type 974 // (result) is 128-bit but the source is 256-bit wide. 975 if (VT.is128BitVector()) 976 setOperationAction(ISD::EXTRACT_SUBVECTOR, SVT, Custom); 977 978 // Do not attempt to custom lower other non-256-bit vectors 979 if (!VT.is256BitVector()) 980 continue; 981 982 setOperationAction(ISD::BUILD_VECTOR, SVT, Custom); 983 setOperationAction(ISD::VECTOR_SHUFFLE, SVT, Custom); 984 setOperationAction(ISD::INSERT_VECTOR_ELT, SVT, Custom); 985 setOperationAction(ISD::EXTRACT_VECTOR_ELT, SVT, Custom); 986 setOperationAction(ISD::INSERT_SUBVECTOR, SVT, Custom); 987 } 988 989 // Promote v32i8, v16i16, v8i32 select, and, or, xor to v4i64. 990 for (unsigned i = (unsigned)MVT::v32i8; i != (unsigned)MVT::v4i64; ++i) { 991 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 992 EVT VT = SVT; 993 994 // Do not attempt to promote non-256-bit vectors 995 if (!VT.is256BitVector()) 996 continue; 997 998 setOperationAction(ISD::AND, SVT, Promote); 999 AddPromotedToType (ISD::AND, SVT, MVT::v4i64); 1000 setOperationAction(ISD::OR, SVT, Promote); 1001 AddPromotedToType (ISD::OR, SVT, MVT::v4i64); 1002 setOperationAction(ISD::XOR, SVT, Promote); 1003 AddPromotedToType (ISD::XOR, SVT, MVT::v4i64); 1004 setOperationAction(ISD::LOAD, SVT, Promote); 1005 AddPromotedToType (ISD::LOAD, SVT, MVT::v4i64); 1006 setOperationAction(ISD::SELECT, SVT, Promote); 1007 AddPromotedToType (ISD::SELECT, SVT, MVT::v4i64); 1008 } 1009 } 1010 1011 // SIGN_EXTEND_INREGs are evaluated by the extend type. Handle the expansion 1012 // of this type with custom code. 1013 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 1014 VT != (unsigned)MVT::LAST_VECTOR_VALUETYPE; VT++) { 1015 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT, Custom); 1016 } 1017 1018 // We want to custom lower some of our intrinsics. 1019 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 1020 1021 1022 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't 1023 // handle type legalization for these operations here. 1024 // 1025 // FIXME: We really should do custom legalization for addition and 1026 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better 1027 // than generic legalization for 64-bit multiplication-with-overflow, though. 1028 for (unsigned i = 0, e = 3+Subtarget->is64Bit(); i != e; ++i) { 1029 // Add/Sub/Mul with overflow operations are custom lowered. 1030 MVT VT = IntVTs[i]; 1031 setOperationAction(ISD::SADDO, VT, Custom); 1032 setOperationAction(ISD::UADDO, VT, Custom); 1033 setOperationAction(ISD::SSUBO, VT, Custom); 1034 setOperationAction(ISD::USUBO, VT, Custom); 1035 setOperationAction(ISD::SMULO, VT, Custom); 1036 setOperationAction(ISD::UMULO, VT, Custom); 1037 } 1038 1039 // There are no 8-bit 3-address imul/mul instructions 1040 setOperationAction(ISD::SMULO, MVT::i8, Expand); 1041 setOperationAction(ISD::UMULO, MVT::i8, Expand); 1042 1043 if (!Subtarget->is64Bit()) { 1044 // These libcalls are not available in 32-bit. 1045 setLibcallName(RTLIB::SHL_I128, 0); 1046 setLibcallName(RTLIB::SRL_I128, 0); 1047 setLibcallName(RTLIB::SRA_I128, 0); 1048 } 1049 1050 // We have target-specific dag combine patterns for the following nodes: 1051 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 1052 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 1053 setTargetDAGCombine(ISD::BUILD_VECTOR); 1054 setTargetDAGCombine(ISD::SELECT); 1055 setTargetDAGCombine(ISD::SHL); 1056 setTargetDAGCombine(ISD::SRA); 1057 setTargetDAGCombine(ISD::SRL); 1058 setTargetDAGCombine(ISD::OR); 1059 setTargetDAGCombine(ISD::AND); 1060 setTargetDAGCombine(ISD::ADD); 1061 setTargetDAGCombine(ISD::SUB); 1062 setTargetDAGCombine(ISD::STORE); 1063 setTargetDAGCombine(ISD::ZERO_EXTEND); 1064 setTargetDAGCombine(ISD::SINT_TO_FP); 1065 if (Subtarget->is64Bit()) 1066 setTargetDAGCombine(ISD::MUL); 1067 1068 computeRegisterProperties(); 1069 1070 // On Darwin, -Os means optimize for size without hurting performance, 1071 // do not reduce the limit. 1072 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 1073 maxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 16 : 8; 1074 maxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores 1075 maxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 8 : 4; 1076 maxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores 1077 maxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 8 : 4; 1078 setPrefLoopAlignment(16); 1079 benefitFromCodePlacementOpt = true; 1080 1081 setPrefFunctionAlignment(4); 1082} 1083 1084 1085MVT::SimpleValueType X86TargetLowering::getSetCCResultType(EVT VT) const { 1086 return MVT::i8; 1087} 1088 1089 1090/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 1091/// the desired ByVal argument alignment. 1092static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) { 1093 if (MaxAlign == 16) 1094 return; 1095 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1096 if (VTy->getBitWidth() == 128) 1097 MaxAlign = 16; 1098 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1099 unsigned EltAlign = 0; 1100 getMaxByValAlign(ATy->getElementType(), EltAlign); 1101 if (EltAlign > MaxAlign) 1102 MaxAlign = EltAlign; 1103 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1104 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 1105 unsigned EltAlign = 0; 1106 getMaxByValAlign(STy->getElementType(i), EltAlign); 1107 if (EltAlign > MaxAlign) 1108 MaxAlign = EltAlign; 1109 if (MaxAlign == 16) 1110 break; 1111 } 1112 } 1113 return; 1114} 1115 1116/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1117/// function arguments in the caller parameter area. For X86, aggregates 1118/// that contain SSE vectors are placed at 16-byte boundaries while the rest 1119/// are at 4-byte boundaries. 1120unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const { 1121 if (Subtarget->is64Bit()) { 1122 // Max of 8 and alignment of type. 1123 unsigned TyAlign = TD->getABITypeAlignment(Ty); 1124 if (TyAlign > 8) 1125 return TyAlign; 1126 return 8; 1127 } 1128 1129 unsigned Align = 4; 1130 if (Subtarget->hasXMM()) 1131 getMaxByValAlign(Ty, Align); 1132 return Align; 1133} 1134 1135/// getOptimalMemOpType - Returns the target specific optimal type for load 1136/// and store operations as a result of memset, memcpy, and memmove 1137/// lowering. If DstAlign is zero that means it's safe to destination 1138/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 1139/// means there isn't a need to check it against alignment requirement, 1140/// probably because the source does not need to be loaded. If 1141/// 'NonScalarIntSafe' is true, that means it's safe to return a 1142/// non-scalar-integer type, e.g. empty string source, constant, or loaded 1143/// from memory. 'MemcpyStrSrc' indicates whether the memcpy source is 1144/// constant so it does not need to be loaded. 1145/// It returns EVT::Other if the type should be determined using generic 1146/// target-independent logic. 1147EVT 1148X86TargetLowering::getOptimalMemOpType(uint64_t Size, 1149 unsigned DstAlign, unsigned SrcAlign, 1150 bool NonScalarIntSafe, 1151 bool MemcpyStrSrc, 1152 MachineFunction &MF) const { 1153 // FIXME: This turns off use of xmm stores for memset/memcpy on targets like 1154 // linux. This is because the stack realignment code can't handle certain 1155 // cases like PR2962. This should be removed when PR2962 is fixed. 1156 const Function *F = MF.getFunction(); 1157 if (NonScalarIntSafe && 1158 !F->hasFnAttr(Attribute::NoImplicitFloat)) { 1159 if (Size >= 16 && 1160 (Subtarget->isUnalignedMemAccessFast() || 1161 ((DstAlign == 0 || DstAlign >= 16) && 1162 (SrcAlign == 0 || SrcAlign >= 16))) && 1163 Subtarget->getStackAlignment() >= 16) { 1164 if (Subtarget->hasSSE2()) 1165 return MVT::v4i32; 1166 if (Subtarget->hasSSE1()) 1167 return MVT::v4f32; 1168 } else if (!MemcpyStrSrc && Size >= 8 && 1169 !Subtarget->is64Bit() && 1170 Subtarget->getStackAlignment() >= 8 && 1171 Subtarget->hasXMMInt()) { 1172 // Do not use f64 to lower memcpy if source is string constant. It's 1173 // better to use i32 to avoid the loads. 1174 return MVT::f64; 1175 } 1176 } 1177 if (Subtarget->is64Bit() && Size >= 8) 1178 return MVT::i64; 1179 return MVT::i32; 1180} 1181 1182/// getJumpTableEncoding - Return the entry encoding for a jump table in the 1183/// current function. The returned value is a member of the 1184/// MachineJumpTableInfo::JTEntryKind enum. 1185unsigned X86TargetLowering::getJumpTableEncoding() const { 1186 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF 1187 // symbol. 1188 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1189 Subtarget->isPICStyleGOT()) 1190 return MachineJumpTableInfo::EK_Custom32; 1191 1192 // Otherwise, use the normal jump table encoding heuristics. 1193 return TargetLowering::getJumpTableEncoding(); 1194} 1195 1196const MCExpr * 1197X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, 1198 const MachineBasicBlock *MBB, 1199 unsigned uid,MCContext &Ctx) const{ 1200 assert(getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1201 Subtarget->isPICStyleGOT()); 1202 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF 1203 // entries. 1204 return MCSymbolRefExpr::Create(MBB->getSymbol(), 1205 MCSymbolRefExpr::VK_GOTOFF, Ctx); 1206} 1207 1208/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 1209/// jumptable. 1210SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 1211 SelectionDAG &DAG) const { 1212 if (!Subtarget->is64Bit()) 1213 // This doesn't have DebugLoc associated with it, but is not really the 1214 // same as a Register. 1215 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy()); 1216 return Table; 1217} 1218 1219/// getPICJumpTableRelocBaseExpr - This returns the relocation base for the 1220/// given PIC jumptable, the same as getPICJumpTableRelocBase, but as an 1221/// MCExpr. 1222const MCExpr *X86TargetLowering:: 1223getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, 1224 MCContext &Ctx) const { 1225 // X86-64 uses RIP relative addressing based on the jump table label. 1226 if (Subtarget->isPICStyleRIPRel()) 1227 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 1228 1229 // Otherwise, the reference is relative to the PIC base. 1230 return MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx); 1231} 1232 1233// FIXME: Why this routine is here? Move to RegInfo! 1234std::pair<const TargetRegisterClass*, uint8_t> 1235X86TargetLowering::findRepresentativeClass(EVT VT) const{ 1236 const TargetRegisterClass *RRC = 0; 1237 uint8_t Cost = 1; 1238 switch (VT.getSimpleVT().SimpleTy) { 1239 default: 1240 return TargetLowering::findRepresentativeClass(VT); 1241 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: 1242 RRC = (Subtarget->is64Bit() 1243 ? X86::GR64RegisterClass : X86::GR32RegisterClass); 1244 break; 1245 case MVT::x86mmx: 1246 RRC = X86::VR64RegisterClass; 1247 break; 1248 case MVT::f32: case MVT::f64: 1249 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: 1250 case MVT::v4f32: case MVT::v2f64: 1251 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: 1252 case MVT::v4f64: 1253 RRC = X86::VR128RegisterClass; 1254 break; 1255 } 1256 return std::make_pair(RRC, Cost); 1257} 1258 1259bool X86TargetLowering::getStackCookieLocation(unsigned &AddressSpace, 1260 unsigned &Offset) const { 1261 if (!Subtarget->isTargetLinux()) 1262 return false; 1263 1264 if (Subtarget->is64Bit()) { 1265 // %fs:0x28, unless we're using a Kernel code model, in which case it's %gs: 1266 Offset = 0x28; 1267 if (getTargetMachine().getCodeModel() == CodeModel::Kernel) 1268 AddressSpace = 256; 1269 else 1270 AddressSpace = 257; 1271 } else { 1272 // %gs:0x14 on i386 1273 Offset = 0x14; 1274 AddressSpace = 256; 1275 } 1276 return true; 1277} 1278 1279 1280//===----------------------------------------------------------------------===// 1281// Return Value Calling Convention Implementation 1282//===----------------------------------------------------------------------===// 1283 1284#include "X86GenCallingConv.inc" 1285 1286bool 1287X86TargetLowering::CanLowerReturn(CallingConv::ID CallConv, 1288 MachineFunction &MF, bool isVarArg, 1289 const SmallVectorImpl<ISD::OutputArg> &Outs, 1290 LLVMContext &Context) const { 1291 SmallVector<CCValAssign, 16> RVLocs; 1292 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1293 RVLocs, Context); 1294 return CCInfo.CheckReturn(Outs, RetCC_X86); 1295} 1296 1297SDValue 1298X86TargetLowering::LowerReturn(SDValue Chain, 1299 CallingConv::ID CallConv, bool isVarArg, 1300 const SmallVectorImpl<ISD::OutputArg> &Outs, 1301 const SmallVectorImpl<SDValue> &OutVals, 1302 DebugLoc dl, SelectionDAG &DAG) const { 1303 MachineFunction &MF = DAG.getMachineFunction(); 1304 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1305 1306 SmallVector<CCValAssign, 16> RVLocs; 1307 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1308 RVLocs, *DAG.getContext()); 1309 CCInfo.AnalyzeReturn(Outs, RetCC_X86); 1310 1311 // Add the regs to the liveout set for the function. 1312 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 1313 for (unsigned i = 0; i != RVLocs.size(); ++i) 1314 if (RVLocs[i].isRegLoc() && !MRI.isLiveOut(RVLocs[i].getLocReg())) 1315 MRI.addLiveOut(RVLocs[i].getLocReg()); 1316 1317 SDValue Flag; 1318 1319 SmallVector<SDValue, 6> RetOps; 1320 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 1321 // Operand #1 = Bytes To Pop 1322 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), 1323 MVT::i16)); 1324 1325 // Copy the result values into the output registers. 1326 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1327 CCValAssign &VA = RVLocs[i]; 1328 assert(VA.isRegLoc() && "Can only return in registers!"); 1329 SDValue ValToCopy = OutVals[i]; 1330 EVT ValVT = ValToCopy.getValueType(); 1331 1332 // If this is x86-64, and we disabled SSE, we can't return FP values, 1333 // or SSE or MMX vectors. 1334 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || 1335 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) && 1336 (Subtarget->is64Bit() && !Subtarget->hasXMM())) { 1337 report_fatal_error("SSE register return with SSE disabled"); 1338 } 1339 // Likewise we can't return F64 values with SSE1 only. gcc does so, but 1340 // llvm-gcc has never done it right and no one has noticed, so this 1341 // should be OK for now. 1342 if (ValVT == MVT::f64 && 1343 (Subtarget->is64Bit() && !Subtarget->hasXMMInt())) 1344 report_fatal_error("SSE2 register return with SSE2 disabled"); 1345 1346 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 1347 // the RET instruction and handled by the FP Stackifier. 1348 if (VA.getLocReg() == X86::ST0 || 1349 VA.getLocReg() == X86::ST1) { 1350 // If this is a copy from an xmm register to ST(0), use an FPExtend to 1351 // change the value to the FP stack register class. 1352 if (isScalarFPTypeInSSEReg(VA.getValVT())) 1353 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); 1354 RetOps.push_back(ValToCopy); 1355 // Don't emit a copytoreg. 1356 continue; 1357 } 1358 1359 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64 1360 // which is returned in RAX / RDX. 1361 if (Subtarget->is64Bit()) { 1362 if (ValVT == MVT::x86mmx) { 1363 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { 1364 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); 1365 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, 1366 ValToCopy); 1367 // If we don't have SSE2 available, convert to v4f32 so the generated 1368 // register is legal. 1369 if (!Subtarget->hasSSE2()) 1370 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); 1371 } 1372 } 1373 } 1374 1375 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag); 1376 Flag = Chain.getValue(1); 1377 } 1378 1379 // The x86-64 ABI for returning structs by value requires that we copy 1380 // the sret argument into %rax for the return. We saved the argument into 1381 // a virtual register in the entry block, so now we copy the value out 1382 // and into %rax. 1383 if (Subtarget->is64Bit() && 1384 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1385 MachineFunction &MF = DAG.getMachineFunction(); 1386 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1387 unsigned Reg = FuncInfo->getSRetReturnReg(); 1388 assert(Reg && 1389 "SRetReturnReg should have been set in LowerFormalArguments()."); 1390 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy()); 1391 1392 Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag); 1393 Flag = Chain.getValue(1); 1394 1395 // RAX now acts like a return value. 1396 MRI.addLiveOut(X86::RAX); 1397 } 1398 1399 RetOps[0] = Chain; // Update chain. 1400 1401 // Add the flag if we have it. 1402 if (Flag.getNode()) 1403 RetOps.push_back(Flag); 1404 1405 return DAG.getNode(X86ISD::RET_FLAG, dl, 1406 MVT::Other, &RetOps[0], RetOps.size()); 1407} 1408 1409bool X86TargetLowering::isUsedByReturnOnly(SDNode *N) const { 1410 if (N->getNumValues() != 1) 1411 return false; 1412 if (!N->hasNUsesOfValue(1, 0)) 1413 return false; 1414 1415 SDNode *Copy = *N->use_begin(); 1416 if (Copy->getOpcode() != ISD::CopyToReg && 1417 Copy->getOpcode() != ISD::FP_EXTEND) 1418 return false; 1419 1420 bool HasRet = false; 1421 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); 1422 UI != UE; ++UI) { 1423 if (UI->getOpcode() != X86ISD::RET_FLAG) 1424 return false; 1425 HasRet = true; 1426 } 1427 1428 return HasRet; 1429} 1430 1431EVT 1432X86TargetLowering::getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT, 1433 ISD::NodeType ExtendKind) const { 1434 MVT ReturnMVT; 1435 // TODO: Is this also valid on 32-bit? 1436 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) 1437 ReturnMVT = MVT::i8; 1438 else 1439 ReturnMVT = MVT::i32; 1440 1441 EVT MinVT = getRegisterType(Context, ReturnMVT); 1442 return VT.bitsLT(MinVT) ? MinVT : VT; 1443} 1444 1445/// LowerCallResult - Lower the result values of a call into the 1446/// appropriate copies out of appropriate physical registers. 1447/// 1448SDValue 1449X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag, 1450 CallingConv::ID CallConv, bool isVarArg, 1451 const SmallVectorImpl<ISD::InputArg> &Ins, 1452 DebugLoc dl, SelectionDAG &DAG, 1453 SmallVectorImpl<SDValue> &InVals) const { 1454 1455 // Assign locations to each value returned by this call. 1456 SmallVector<CCValAssign, 16> RVLocs; 1457 bool Is64Bit = Subtarget->is64Bit(); 1458 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 1459 getTargetMachine(), RVLocs, *DAG.getContext()); 1460 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 1461 1462 // Copy all of the result registers out of their specified physreg. 1463 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1464 CCValAssign &VA = RVLocs[i]; 1465 EVT CopyVT = VA.getValVT(); 1466 1467 // If this is x86-64, and we disabled SSE, we can't return FP values 1468 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) && 1469 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasXMM())) { 1470 report_fatal_error("SSE register return with SSE disabled"); 1471 } 1472 1473 SDValue Val; 1474 1475 // If this is a call to a function that returns an fp value on the floating 1476 // point stack, we must guarantee the the value is popped from the stack, so 1477 // a CopyFromReg is not good enough - the copy instruction may be eliminated 1478 // if the return value is not used. We use the FpPOP_RETVAL instruction 1479 // instead. 1480 if (VA.getLocReg() == X86::ST0 || VA.getLocReg() == X86::ST1) { 1481 // If we prefer to use the value in xmm registers, copy it out as f80 and 1482 // use a truncate to move it from fp stack reg to xmm reg. 1483 if (isScalarFPTypeInSSEReg(VA.getValVT())) CopyVT = MVT::f80; 1484 SDValue Ops[] = { Chain, InFlag }; 1485 Chain = SDValue(DAG.getMachineNode(X86::FpPOP_RETVAL, dl, CopyVT, 1486 MVT::Other, MVT::Glue, Ops, 2), 1); 1487 Val = Chain.getValue(0); 1488 1489 // Round the f80 to the right size, which also moves it to the appropriate 1490 // xmm register. 1491 if (CopyVT != VA.getValVT()) 1492 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, 1493 // This truncation won't change the value. 1494 DAG.getIntPtrConstant(1)); 1495 } else { 1496 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 1497 CopyVT, InFlag).getValue(1); 1498 Val = Chain.getValue(0); 1499 } 1500 InFlag = Chain.getValue(2); 1501 InVals.push_back(Val); 1502 } 1503 1504 return Chain; 1505} 1506 1507 1508//===----------------------------------------------------------------------===// 1509// C & StdCall & Fast Calling Convention implementation 1510//===----------------------------------------------------------------------===// 1511// StdCall calling convention seems to be standard for many Windows' API 1512// routines and around. It differs from C calling convention just a little: 1513// callee should clean up the stack, not caller. Symbols should be also 1514// decorated in some fancy way :) It doesn't support any vector arguments. 1515// For info on fast calling convention see Fast Calling Convention (tail call) 1516// implementation LowerX86_32FastCCCallTo. 1517 1518/// CallIsStructReturn - Determines whether a call uses struct return 1519/// semantics. 1520static bool CallIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) { 1521 if (Outs.empty()) 1522 return false; 1523 1524 return Outs[0].Flags.isSRet(); 1525} 1526 1527/// ArgsAreStructReturn - Determines whether a function uses struct 1528/// return semantics. 1529static bool 1530ArgsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) { 1531 if (Ins.empty()) 1532 return false; 1533 1534 return Ins[0].Flags.isSRet(); 1535} 1536 1537/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1538/// by "Src" to address "Dst" with size and alignment information specified by 1539/// the specific parameter attribute. The copy will be passed as a byval 1540/// function parameter. 1541static SDValue 1542CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 1543 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, 1544 DebugLoc dl) { 1545 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1546 1547 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 1548 /*isVolatile*/false, /*AlwaysInline=*/true, 1549 MachinePointerInfo(), MachinePointerInfo()); 1550} 1551 1552/// IsTailCallConvention - Return true if the calling convention is one that 1553/// supports tail call optimization. 1554static bool IsTailCallConvention(CallingConv::ID CC) { 1555 return (CC == CallingConv::Fast || CC == CallingConv::GHC); 1556} 1557 1558bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 1559 if (!CI->isTailCall()) 1560 return false; 1561 1562 CallSite CS(CI); 1563 CallingConv::ID CalleeCC = CS.getCallingConv(); 1564 if (!IsTailCallConvention(CalleeCC) && CalleeCC != CallingConv::C) 1565 return false; 1566 1567 return true; 1568} 1569 1570/// FuncIsMadeTailCallSafe - Return true if the function is being made into 1571/// a tailcall target by changing its ABI. 1572static bool FuncIsMadeTailCallSafe(CallingConv::ID CC) { 1573 return GuaranteedTailCallOpt && IsTailCallConvention(CC); 1574} 1575 1576SDValue 1577X86TargetLowering::LowerMemArgument(SDValue Chain, 1578 CallingConv::ID CallConv, 1579 const SmallVectorImpl<ISD::InputArg> &Ins, 1580 DebugLoc dl, SelectionDAG &DAG, 1581 const CCValAssign &VA, 1582 MachineFrameInfo *MFI, 1583 unsigned i) const { 1584 // Create the nodes corresponding to a load from this parameter slot. 1585 ISD::ArgFlagsTy Flags = Ins[i].Flags; 1586 bool AlwaysUseMutable = FuncIsMadeTailCallSafe(CallConv); 1587 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1588 EVT ValVT; 1589 1590 // If value is passed by pointer we have address passed instead of the value 1591 // itself. 1592 if (VA.getLocInfo() == CCValAssign::Indirect) 1593 ValVT = VA.getLocVT(); 1594 else 1595 ValVT = VA.getValVT(); 1596 1597 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1598 // changed with more analysis. 1599 // In case of tail call optimization mark all arguments mutable. Since they 1600 // could be overwritten by lowering of arguments in case of a tail call. 1601 if (Flags.isByVal()) { 1602 unsigned Bytes = Flags.getByValSize(); 1603 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects. 1604 int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable); 1605 return DAG.getFrameIndex(FI, getPointerTy()); 1606 } else { 1607 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8, 1608 VA.getLocMemOffset(), isImmutable); 1609 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 1610 return DAG.getLoad(ValVT, dl, Chain, FIN, 1611 MachinePointerInfo::getFixedStack(FI), 1612 false, false, 0); 1613 } 1614} 1615 1616SDValue 1617X86TargetLowering::LowerFormalArguments(SDValue Chain, 1618 CallingConv::ID CallConv, 1619 bool isVarArg, 1620 const SmallVectorImpl<ISD::InputArg> &Ins, 1621 DebugLoc dl, 1622 SelectionDAG &DAG, 1623 SmallVectorImpl<SDValue> &InVals) 1624 const { 1625 MachineFunction &MF = DAG.getMachineFunction(); 1626 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1627 1628 const Function* Fn = MF.getFunction(); 1629 if (Fn->hasExternalLinkage() && 1630 Subtarget->isTargetCygMing() && 1631 Fn->getName() == "main") 1632 FuncInfo->setForceFramePointer(true); 1633 1634 MachineFrameInfo *MFI = MF.getFrameInfo(); 1635 bool Is64Bit = Subtarget->is64Bit(); 1636 bool IsWin64 = Subtarget->isTargetWin64(); 1637 1638 assert(!(isVarArg && IsTailCallConvention(CallConv)) && 1639 "Var args not supported with calling convention fastcc or ghc"); 1640 1641 // Assign locations to all of the incoming arguments. 1642 SmallVector<CCValAssign, 16> ArgLocs; 1643 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1644 ArgLocs, *DAG.getContext()); 1645 1646 // Allocate shadow area for Win64 1647 if (IsWin64) { 1648 CCInfo.AllocateStack(32, 8); 1649 } 1650 1651 CCInfo.AnalyzeFormalArguments(Ins, CC_X86); 1652 1653 unsigned LastVal = ~0U; 1654 SDValue ArgValue; 1655 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1656 CCValAssign &VA = ArgLocs[i]; 1657 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1658 // places. 1659 assert(VA.getValNo() != LastVal && 1660 "Don't support value assigned to multiple locs yet"); 1661 LastVal = VA.getValNo(); 1662 1663 if (VA.isRegLoc()) { 1664 EVT RegVT = VA.getLocVT(); 1665 TargetRegisterClass *RC = NULL; 1666 if (RegVT == MVT::i32) 1667 RC = X86::GR32RegisterClass; 1668 else if (Is64Bit && RegVT == MVT::i64) 1669 RC = X86::GR64RegisterClass; 1670 else if (RegVT == MVT::f32) 1671 RC = X86::FR32RegisterClass; 1672 else if (RegVT == MVT::f64) 1673 RC = X86::FR64RegisterClass; 1674 else if (RegVT.isVector() && RegVT.getSizeInBits() == 256) 1675 RC = X86::VR256RegisterClass; 1676 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1677 RC = X86::VR128RegisterClass; 1678 else if (RegVT == MVT::x86mmx) 1679 RC = X86::VR64RegisterClass; 1680 else 1681 llvm_unreachable("Unknown argument type!"); 1682 1683 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 1684 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 1685 1686 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1687 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1688 // right size. 1689 if (VA.getLocInfo() == CCValAssign::SExt) 1690 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 1691 DAG.getValueType(VA.getValVT())); 1692 else if (VA.getLocInfo() == CCValAssign::ZExt) 1693 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 1694 DAG.getValueType(VA.getValVT())); 1695 else if (VA.getLocInfo() == CCValAssign::BCvt) 1696 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); 1697 1698 if (VA.isExtInLoc()) { 1699 // Handle MMX values passed in XMM regs. 1700 if (RegVT.isVector()) { 1701 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), 1702 ArgValue); 1703 } else 1704 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 1705 } 1706 } else { 1707 assert(VA.isMemLoc()); 1708 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i); 1709 } 1710 1711 // If value is passed via pointer - do a load. 1712 if (VA.getLocInfo() == CCValAssign::Indirect) 1713 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, 1714 MachinePointerInfo(), false, false, 0); 1715 1716 InVals.push_back(ArgValue); 1717 } 1718 1719 // The x86-64 ABI for returning structs by value requires that we copy 1720 // the sret argument into %rax for the return. Save the argument into 1721 // a virtual register so that we can access it from the return points. 1722 if (Is64Bit && MF.getFunction()->hasStructRetAttr()) { 1723 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1724 unsigned Reg = FuncInfo->getSRetReturnReg(); 1725 if (!Reg) { 1726 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1727 FuncInfo->setSRetReturnReg(Reg); 1728 } 1729 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]); 1730 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); 1731 } 1732 1733 unsigned StackSize = CCInfo.getNextStackOffset(); 1734 // Align stack specially for tail calls. 1735 if (FuncIsMadeTailCallSafe(CallConv)) 1736 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1737 1738 // If the function takes variable number of arguments, make a frame index for 1739 // the start of the first vararg value... for expansion of llvm.va_start. 1740 if (isVarArg) { 1741 if (Is64Bit || (CallConv != CallingConv::X86_FastCall && 1742 CallConv != CallingConv::X86_ThisCall)) { 1743 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize,true)); 1744 } 1745 if (Is64Bit) { 1746 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1747 1748 // FIXME: We should really autogenerate these arrays 1749 static const unsigned GPR64ArgRegsWin64[] = { 1750 X86::RCX, X86::RDX, X86::R8, X86::R9 1751 }; 1752 static const unsigned GPR64ArgRegs64Bit[] = { 1753 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1754 }; 1755 static const unsigned XMMArgRegs64Bit[] = { 1756 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1757 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1758 }; 1759 const unsigned *GPR64ArgRegs; 1760 unsigned NumXMMRegs = 0; 1761 1762 if (IsWin64) { 1763 // The XMM registers which might contain var arg parameters are shadowed 1764 // in their paired GPR. So we only need to save the GPR to their home 1765 // slots. 1766 TotalNumIntRegs = 4; 1767 GPR64ArgRegs = GPR64ArgRegsWin64; 1768 } else { 1769 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1770 GPR64ArgRegs = GPR64ArgRegs64Bit; 1771 1772 NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs64Bit, TotalNumXMMRegs); 1773 } 1774 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1775 TotalNumIntRegs); 1776 1777 bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat); 1778 assert(!(NumXMMRegs && !Subtarget->hasXMM()) && 1779 "SSE register cannot be used when SSE is disabled!"); 1780 assert(!(NumXMMRegs && UseSoftFloat && NoImplicitFloatOps) && 1781 "SSE register cannot be used when SSE is disabled!"); 1782 if (UseSoftFloat || NoImplicitFloatOps || !Subtarget->hasXMM()) 1783 // Kernel mode asks for SSE to be disabled, so don't push them 1784 // on the stack. 1785 TotalNumXMMRegs = 0; 1786 1787 if (IsWin64) { 1788 const TargetFrameLowering &TFI = *getTargetMachine().getFrameLowering(); 1789 // Get to the caller-allocated home save location. Add 8 to account 1790 // for the return address. 1791 int HomeOffset = TFI.getOffsetOfLocalArea() + 8; 1792 FuncInfo->setRegSaveFrameIndex( 1793 MFI->CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false)); 1794 // Fixup to set vararg frame on shadow area (4 x i64). 1795 if (NumIntRegs < 4) 1796 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex()); 1797 } else { 1798 // For X86-64, if there are vararg parameters that are passed via 1799 // registers, then we must store them to their spots on the stack so they 1800 // may be loaded by deferencing the result of va_next. 1801 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8); 1802 FuncInfo->setVarArgsFPOffset(TotalNumIntRegs * 8 + NumXMMRegs * 16); 1803 FuncInfo->setRegSaveFrameIndex( 1804 MFI->CreateStackObject(TotalNumIntRegs * 8 + TotalNumXMMRegs * 16, 16, 1805 false)); 1806 } 1807 1808 // Store the integer parameter registers. 1809 SmallVector<SDValue, 8> MemOps; 1810 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), 1811 getPointerTy()); 1812 unsigned Offset = FuncInfo->getVarArgsGPOffset(); 1813 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1814 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN, 1815 DAG.getIntPtrConstant(Offset)); 1816 unsigned VReg = MF.addLiveIn(GPR64ArgRegs[NumIntRegs], 1817 X86::GR64RegisterClass); 1818 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 1819 SDValue Store = 1820 DAG.getStore(Val.getValue(1), dl, Val, FIN, 1821 MachinePointerInfo::getFixedStack( 1822 FuncInfo->getRegSaveFrameIndex(), Offset), 1823 false, false, 0); 1824 MemOps.push_back(Store); 1825 Offset += 8; 1826 } 1827 1828 if (TotalNumXMMRegs != 0 && NumXMMRegs != TotalNumXMMRegs) { 1829 // Now store the XMM (fp + vector) parameter registers. 1830 SmallVector<SDValue, 11> SaveXMMOps; 1831 SaveXMMOps.push_back(Chain); 1832 1833 unsigned AL = MF.addLiveIn(X86::AL, X86::GR8RegisterClass); 1834 SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8); 1835 SaveXMMOps.push_back(ALVal); 1836 1837 SaveXMMOps.push_back(DAG.getIntPtrConstant( 1838 FuncInfo->getRegSaveFrameIndex())); 1839 SaveXMMOps.push_back(DAG.getIntPtrConstant( 1840 FuncInfo->getVarArgsFPOffset())); 1841 1842 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1843 unsigned VReg = MF.addLiveIn(XMMArgRegs64Bit[NumXMMRegs], 1844 X86::VR128RegisterClass); 1845 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32); 1846 SaveXMMOps.push_back(Val); 1847 } 1848 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, 1849 MVT::Other, 1850 &SaveXMMOps[0], SaveXMMOps.size())); 1851 } 1852 1853 if (!MemOps.empty()) 1854 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1855 &MemOps[0], MemOps.size()); 1856 } 1857 } 1858 1859 // Some CCs need callee pop. 1860 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, GuaranteedTailCallOpt)) { 1861 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything. 1862 } else { 1863 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing. 1864 // If this is an sret function, the return should pop the hidden pointer. 1865 if (!Is64Bit && !IsTailCallConvention(CallConv) && ArgsAreStructReturn(Ins)) 1866 FuncInfo->setBytesToPopOnReturn(4); 1867 } 1868 1869 if (!Is64Bit) { 1870 // RegSaveFrameIndex is X86-64 only. 1871 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); 1872 if (CallConv == CallingConv::X86_FastCall || 1873 CallConv == CallingConv::X86_ThisCall) 1874 // fastcc functions can't have varargs. 1875 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA); 1876 } 1877 1878 return Chain; 1879} 1880 1881SDValue 1882X86TargetLowering::LowerMemOpCallTo(SDValue Chain, 1883 SDValue StackPtr, SDValue Arg, 1884 DebugLoc dl, SelectionDAG &DAG, 1885 const CCValAssign &VA, 1886 ISD::ArgFlagsTy Flags) const { 1887 unsigned LocMemOffset = VA.getLocMemOffset(); 1888 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1889 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); 1890 if (Flags.isByVal()) 1891 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); 1892 1893 return DAG.getStore(Chain, dl, Arg, PtrOff, 1894 MachinePointerInfo::getStack(LocMemOffset), 1895 false, false, 0); 1896} 1897 1898/// EmitTailCallLoadRetAddr - Emit a load of return address if tail call 1899/// optimization is performed and it is required. 1900SDValue 1901X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1902 SDValue &OutRetAddr, SDValue Chain, 1903 bool IsTailCall, bool Is64Bit, 1904 int FPDiff, DebugLoc dl) const { 1905 // Adjust the Return address stack slot. 1906 EVT VT = getPointerTy(); 1907 OutRetAddr = getReturnAddressFrameIndex(DAG); 1908 1909 // Load the "old" Return address. 1910 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(), 1911 false, false, 0); 1912 return SDValue(OutRetAddr.getNode(), 1); 1913} 1914 1915/// EmitTailCallStoreRetAddr - Emit a store of the return address if tail call 1916/// optimization is performed and it is required (FPDiff!=0). 1917static SDValue 1918EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1919 SDValue Chain, SDValue RetAddrFrIdx, 1920 bool Is64Bit, int FPDiff, DebugLoc dl) { 1921 // Store the return address to the appropriate stack slot. 1922 if (!FPDiff) return Chain; 1923 // Calculate the new stack slot for the return address. 1924 int SlotSize = Is64Bit ? 8 : 4; 1925 int NewReturnAddrFI = 1926 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize, false); 1927 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1928 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1929 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, 1930 MachinePointerInfo::getFixedStack(NewReturnAddrFI), 1931 false, false, 0); 1932 return Chain; 1933} 1934 1935SDValue 1936X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee, 1937 CallingConv::ID CallConv, bool isVarArg, 1938 bool &isTailCall, 1939 const SmallVectorImpl<ISD::OutputArg> &Outs, 1940 const SmallVectorImpl<SDValue> &OutVals, 1941 const SmallVectorImpl<ISD::InputArg> &Ins, 1942 DebugLoc dl, SelectionDAG &DAG, 1943 SmallVectorImpl<SDValue> &InVals) const { 1944 MachineFunction &MF = DAG.getMachineFunction(); 1945 bool Is64Bit = Subtarget->is64Bit(); 1946 bool IsWin64 = Subtarget->isTargetWin64(); 1947 bool IsStructRet = CallIsStructReturn(Outs); 1948 bool IsSibcall = false; 1949 1950 if (isTailCall) { 1951 // Check if it's really possible to do a tail call. 1952 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, 1953 isVarArg, IsStructRet, MF.getFunction()->hasStructRetAttr(), 1954 Outs, OutVals, Ins, DAG); 1955 1956 // Sibcalls are automatically detected tailcalls which do not require 1957 // ABI changes. 1958 if (!GuaranteedTailCallOpt && isTailCall) 1959 IsSibcall = true; 1960 1961 if (isTailCall) 1962 ++NumTailCalls; 1963 } 1964 1965 assert(!(isVarArg && IsTailCallConvention(CallConv)) && 1966 "Var args not supported with calling convention fastcc or ghc"); 1967 1968 // Analyze operands of the call, assigning locations to each operand. 1969 SmallVector<CCValAssign, 16> ArgLocs; 1970 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1971 ArgLocs, *DAG.getContext()); 1972 1973 // Allocate shadow area for Win64 1974 if (IsWin64) { 1975 CCInfo.AllocateStack(32, 8); 1976 } 1977 1978 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 1979 1980 // Get a count of how many bytes are to be pushed on the stack. 1981 unsigned NumBytes = CCInfo.getNextStackOffset(); 1982 if (IsSibcall) 1983 // This is a sibcall. The memory operands are available in caller's 1984 // own caller's stack. 1985 NumBytes = 0; 1986 else if (GuaranteedTailCallOpt && IsTailCallConvention(CallConv)) 1987 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1988 1989 int FPDiff = 0; 1990 if (isTailCall && !IsSibcall) { 1991 // Lower arguments at fp - stackoffset + fpdiff. 1992 unsigned NumBytesCallerPushed = 1993 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1994 FPDiff = NumBytesCallerPushed - NumBytes; 1995 1996 // Set the delta of movement of the returnaddr stackslot. 1997 // But only set if delta is greater than previous delta. 1998 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1999 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 2000 } 2001 2002 if (!IsSibcall) 2003 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true)); 2004 2005 SDValue RetAddrFrIdx; 2006 // Load return address for tail calls. 2007 if (isTailCall && FPDiff) 2008 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall, 2009 Is64Bit, FPDiff, dl); 2010 2011 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 2012 SmallVector<SDValue, 8> MemOpChains; 2013 SDValue StackPtr; 2014 2015 // Walk the register/memloc assignments, inserting copies/loads. In the case 2016 // of tail call optimization arguments are handle later. 2017 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2018 CCValAssign &VA = ArgLocs[i]; 2019 EVT RegVT = VA.getLocVT(); 2020 SDValue Arg = OutVals[i]; 2021 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2022 bool isByVal = Flags.isByVal(); 2023 2024 // Promote the value if needed. 2025 switch (VA.getLocInfo()) { 2026 default: llvm_unreachable("Unknown loc info!"); 2027 case CCValAssign::Full: break; 2028 case CCValAssign::SExt: 2029 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); 2030 break; 2031 case CCValAssign::ZExt: 2032 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); 2033 break; 2034 case CCValAssign::AExt: 2035 if (RegVT.isVector() && RegVT.getSizeInBits() == 128) { 2036 // Special case: passing MMX values in XMM registers. 2037 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 2038 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); 2039 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); 2040 } else 2041 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); 2042 break; 2043 case CCValAssign::BCvt: 2044 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg); 2045 break; 2046 case CCValAssign::Indirect: { 2047 // Store the argument. 2048 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 2049 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 2050 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot, 2051 MachinePointerInfo::getFixedStack(FI), 2052 false, false, 0); 2053 Arg = SpillSlot; 2054 break; 2055 } 2056 } 2057 2058 if (VA.isRegLoc()) { 2059 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 2060 if (isVarArg && IsWin64) { 2061 // Win64 ABI requires argument XMM reg to be copied to the corresponding 2062 // shadow reg if callee is a varargs function. 2063 unsigned ShadowReg = 0; 2064 switch (VA.getLocReg()) { 2065 case X86::XMM0: ShadowReg = X86::RCX; break; 2066 case X86::XMM1: ShadowReg = X86::RDX; break; 2067 case X86::XMM2: ShadowReg = X86::R8; break; 2068 case X86::XMM3: ShadowReg = X86::R9; break; 2069 } 2070 if (ShadowReg) 2071 RegsToPass.push_back(std::make_pair(ShadowReg, Arg)); 2072 } 2073 } else if (!IsSibcall && (!isTailCall || isByVal)) { 2074 assert(VA.isMemLoc()); 2075 if (StackPtr.getNode() == 0) 2076 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy()); 2077 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, 2078 dl, DAG, VA, Flags)); 2079 } 2080 } 2081 2082 if (!MemOpChains.empty()) 2083 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 2084 &MemOpChains[0], MemOpChains.size()); 2085 2086 // Build a sequence of copy-to-reg nodes chained together with token chain 2087 // and flag operands which copy the outgoing args into registers. 2088 SDValue InFlag; 2089 // Tail call byval lowering might overwrite argument registers so in case of 2090 // tail call optimization the copies to registers are lowered later. 2091 if (!isTailCall) 2092 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 2093 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 2094 RegsToPass[i].second, InFlag); 2095 InFlag = Chain.getValue(1); 2096 } 2097 2098 if (Subtarget->isPICStyleGOT()) { 2099 // ELF / PIC requires GOT in the EBX register before function calls via PLT 2100 // GOT pointer. 2101 if (!isTailCall) { 2102 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX, 2103 DAG.getNode(X86ISD::GlobalBaseReg, 2104 DebugLoc(), getPointerTy()), 2105 InFlag); 2106 InFlag = Chain.getValue(1); 2107 } else { 2108 // If we are tail calling and generating PIC/GOT style code load the 2109 // address of the callee into ECX. The value in ecx is used as target of 2110 // the tail jump. This is done to circumvent the ebx/callee-saved problem 2111 // for tail calls on PIC/GOT architectures. Normally we would just put the 2112 // address of GOT into ebx and then call target@PLT. But for tail calls 2113 // ebx would be restored (since ebx is callee saved) before jumping to the 2114 // target@PLT. 2115 2116 // Note: The actual moving to ECX is done further down. 2117 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 2118 if (G && !G->getGlobal()->hasHiddenVisibility() && 2119 !G->getGlobal()->hasProtectedVisibility()) 2120 Callee = LowerGlobalAddress(Callee, DAG); 2121 else if (isa<ExternalSymbolSDNode>(Callee)) 2122 Callee = LowerExternalSymbol(Callee, DAG); 2123 } 2124 } 2125 2126 if (Is64Bit && isVarArg && !IsWin64) { 2127 // From AMD64 ABI document: 2128 // For calls that may call functions that use varargs or stdargs 2129 // (prototype-less calls or calls to functions containing ellipsis (...) in 2130 // the declaration) %al is used as hidden argument to specify the number 2131 // of SSE registers used. The contents of %al do not need to match exactly 2132 // the number of registers, but must be an ubound on the number of SSE 2133 // registers used and is in the range 0 - 8 inclusive. 2134 2135 // Count the number of XMM registers allocated. 2136 static const unsigned XMMArgRegs[] = { 2137 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 2138 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 2139 }; 2140 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 2141 assert((Subtarget->hasXMM() || !NumXMMRegs) 2142 && "SSE registers cannot be used when SSE is disabled"); 2143 2144 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, 2145 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 2146 InFlag = Chain.getValue(1); 2147 } 2148 2149 2150 // For tail calls lower the arguments to the 'real' stack slot. 2151 if (isTailCall) { 2152 // Force all the incoming stack arguments to be loaded from the stack 2153 // before any new outgoing arguments are stored to the stack, because the 2154 // outgoing stack slots may alias the incoming argument stack slots, and 2155 // the alias isn't otherwise explicit. This is slightly more conservative 2156 // than necessary, because it means that each store effectively depends 2157 // on every argument instead of just those arguments it would clobber. 2158 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain); 2159 2160 SmallVector<SDValue, 8> MemOpChains2; 2161 SDValue FIN; 2162 int FI = 0; 2163 // Do not flag preceding copytoreg stuff together with the following stuff. 2164 InFlag = SDValue(); 2165 if (GuaranteedTailCallOpt) { 2166 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2167 CCValAssign &VA = ArgLocs[i]; 2168 if (VA.isRegLoc()) 2169 continue; 2170 assert(VA.isMemLoc()); 2171 SDValue Arg = OutVals[i]; 2172 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2173 // Create frame index. 2174 int32_t Offset = VA.getLocMemOffset()+FPDiff; 2175 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 2176 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); 2177 FIN = DAG.getFrameIndex(FI, getPointerTy()); 2178 2179 if (Flags.isByVal()) { 2180 // Copy relative to framepointer. 2181 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 2182 if (StackPtr.getNode() == 0) 2183 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, 2184 getPointerTy()); 2185 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source); 2186 2187 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, 2188 ArgChain, 2189 Flags, DAG, dl)); 2190 } else { 2191 // Store relative to framepointer. 2192 MemOpChains2.push_back( 2193 DAG.getStore(ArgChain, dl, Arg, FIN, 2194 MachinePointerInfo::getFixedStack(FI), 2195 false, false, 0)); 2196 } 2197 } 2198 } 2199 2200 if (!MemOpChains2.empty()) 2201 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 2202 &MemOpChains2[0], MemOpChains2.size()); 2203 2204 // Copy arguments to their registers. 2205 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 2206 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 2207 RegsToPass[i].second, InFlag); 2208 InFlag = Chain.getValue(1); 2209 } 2210 InFlag =SDValue(); 2211 2212 // Store the return address to the appropriate stack slot. 2213 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 2214 FPDiff, dl); 2215 } 2216 2217 if (getTargetMachine().getCodeModel() == CodeModel::Large) { 2218 assert(Is64Bit && "Large code model is only legal in 64-bit mode."); 2219 // In the 64-bit large code model, we have to make all calls 2220 // through a register, since the call instruction's 32-bit 2221 // pc-relative offset may not be large enough to hold the whole 2222 // address. 2223 } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 2224 // If the callee is a GlobalAddress node (quite common, every direct call 2225 // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack 2226 // it. 2227 2228 // We should use extra load for direct calls to dllimported functions in 2229 // non-JIT mode. 2230 const GlobalValue *GV = G->getGlobal(); 2231 if (!GV->hasDLLImportLinkage()) { 2232 unsigned char OpFlags = 0; 2233 bool ExtraLoad = false; 2234 unsigned WrapperKind = ISD::DELETED_NODE; 2235 2236 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to 2237 // external symbols most go through the PLT in PIC mode. If the symbol 2238 // has hidden or protected visibility, or if it is static or local, then 2239 // we don't need to use the PLT - we can directly call it. 2240 if (Subtarget->isTargetELF() && 2241 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 2242 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) { 2243 OpFlags = X86II::MO_PLT; 2244 } else if (Subtarget->isPICStyleStubAny() && 2245 (GV->isDeclaration() || GV->isWeakForLinker()) && 2246 (!Subtarget->getTargetTriple().isMacOSX() || 2247 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) { 2248 // PC-relative references to external symbols should go through $stub, 2249 // unless we're building with the leopard linker or later, which 2250 // automatically synthesizes these stubs. 2251 OpFlags = X86II::MO_DARWIN_STUB; 2252 } else if (Subtarget->isPICStyleRIPRel() && 2253 isa<Function>(GV) && 2254 cast<Function>(GV)->hasFnAttr(Attribute::NonLazyBind)) { 2255 // If the function is marked as non-lazy, generate an indirect call 2256 // which loads from the GOT directly. This avoids runtime overhead 2257 // at the cost of eager binding (and one extra byte of encoding). 2258 OpFlags = X86II::MO_GOTPCREL; 2259 WrapperKind = X86ISD::WrapperRIP; 2260 ExtraLoad = true; 2261 } 2262 2263 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 2264 G->getOffset(), OpFlags); 2265 2266 // Add a wrapper if needed. 2267 if (WrapperKind != ISD::DELETED_NODE) 2268 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee); 2269 // Add extra indirection if needed. 2270 if (ExtraLoad) 2271 Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee, 2272 MachinePointerInfo::getGOT(), 2273 false, false, 0); 2274 } 2275 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 2276 unsigned char OpFlags = 0; 2277 2278 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to 2279 // external symbols should go through the PLT. 2280 if (Subtarget->isTargetELF() && 2281 getTargetMachine().getRelocationModel() == Reloc::PIC_) { 2282 OpFlags = X86II::MO_PLT; 2283 } else if (Subtarget->isPICStyleStubAny() && 2284 (!Subtarget->getTargetTriple().isMacOSX() || 2285 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) { 2286 // PC-relative references to external symbols should go through $stub, 2287 // unless we're building with the leopard linker or later, which 2288 // automatically synthesizes these stubs. 2289 OpFlags = X86II::MO_DARWIN_STUB; 2290 } 2291 2292 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(), 2293 OpFlags); 2294 } 2295 2296 // Returns a chain & a flag for retval copy to use. 2297 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2298 SmallVector<SDValue, 8> Ops; 2299 2300 if (!IsSibcall && isTailCall) { 2301 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), 2302 DAG.getIntPtrConstant(0, true), InFlag); 2303 InFlag = Chain.getValue(1); 2304 } 2305 2306 Ops.push_back(Chain); 2307 Ops.push_back(Callee); 2308 2309 if (isTailCall) 2310 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 2311 2312 // Add argument registers to the end of the list so that they are known live 2313 // into the call. 2314 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 2315 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 2316 RegsToPass[i].second.getValueType())); 2317 2318 // Add an implicit use GOT pointer in EBX. 2319 if (!isTailCall && Subtarget->isPICStyleGOT()) 2320 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 2321 2322 // Add an implicit use of AL for non-Windows x86 64-bit vararg functions. 2323 if (Is64Bit && isVarArg && !IsWin64) 2324 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 2325 2326 if (InFlag.getNode()) 2327 Ops.push_back(InFlag); 2328 2329 if (isTailCall) { 2330 // We used to do: 2331 //// If this is the first return lowered for this function, add the regs 2332 //// to the liveout set for the function. 2333 // This isn't right, although it's probably harmless on x86; liveouts 2334 // should be computed from returns not tail calls. Consider a void 2335 // function making a tail call to a function returning int. 2336 return DAG.getNode(X86ISD::TC_RETURN, dl, 2337 NodeTys, &Ops[0], Ops.size()); 2338 } 2339 2340 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size()); 2341 InFlag = Chain.getValue(1); 2342 2343 // Create the CALLSEQ_END node. 2344 unsigned NumBytesForCalleeToPush; 2345 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, GuaranteedTailCallOpt)) 2346 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 2347 else if (!Is64Bit && !IsTailCallConvention(CallConv) && IsStructRet) 2348 // If this is a call to a struct-return function, the callee 2349 // pops the hidden struct pointer, so we have to push it back. 2350 // This is common for Darwin/X86, Linux & Mingw32 targets. 2351 NumBytesForCalleeToPush = 4; 2352 else 2353 NumBytesForCalleeToPush = 0; // Callee pops nothing. 2354 2355 // Returns a flag for retval copy to use. 2356 if (!IsSibcall) { 2357 Chain = DAG.getCALLSEQ_END(Chain, 2358 DAG.getIntPtrConstant(NumBytes, true), 2359 DAG.getIntPtrConstant(NumBytesForCalleeToPush, 2360 true), 2361 InFlag); 2362 InFlag = Chain.getValue(1); 2363 } 2364 2365 // Handle result values, copying them out of physregs into vregs that we 2366 // return. 2367 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 2368 Ins, dl, DAG, InVals); 2369} 2370 2371 2372//===----------------------------------------------------------------------===// 2373// Fast Calling Convention (tail call) implementation 2374//===----------------------------------------------------------------------===// 2375 2376// Like std call, callee cleans arguments, convention except that ECX is 2377// reserved for storing the tail called function address. Only 2 registers are 2378// free for argument passing (inreg). Tail call optimization is performed 2379// provided: 2380// * tailcallopt is enabled 2381// * caller/callee are fastcc 2382// On X86_64 architecture with GOT-style position independent code only local 2383// (within module) calls are supported at the moment. 2384// To keep the stack aligned according to platform abi the function 2385// GetAlignedArgumentStackSize ensures that argument delta is always multiples 2386// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 2387// If a tail called function callee has more arguments than the caller the 2388// caller needs to make sure that there is room to move the RETADDR to. This is 2389// achieved by reserving an area the size of the argument delta right after the 2390// original REtADDR, but before the saved framepointer or the spilled registers 2391// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 2392// stack layout: 2393// arg1 2394// arg2 2395// RETADDR 2396// [ new RETADDR 2397// move area ] 2398// (possible EBP) 2399// ESI 2400// EDI 2401// local1 .. 2402 2403/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 2404/// for a 16 byte align requirement. 2405unsigned 2406X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 2407 SelectionDAG& DAG) const { 2408 MachineFunction &MF = DAG.getMachineFunction(); 2409 const TargetMachine &TM = MF.getTarget(); 2410 const TargetFrameLowering &TFI = *TM.getFrameLowering(); 2411 unsigned StackAlignment = TFI.getStackAlignment(); 2412 uint64_t AlignMask = StackAlignment - 1; 2413 int64_t Offset = StackSize; 2414 uint64_t SlotSize = TD->getPointerSize(); 2415 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 2416 // Number smaller than 12 so just add the difference. 2417 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 2418 } else { 2419 // Mask out lower bits, add stackalignment once plus the 12 bytes. 2420 Offset = ((~AlignMask) & Offset) + StackAlignment + 2421 (StackAlignment-SlotSize); 2422 } 2423 return Offset; 2424} 2425 2426/// MatchingStackOffset - Return true if the given stack call argument is 2427/// already available in the same position (relatively) of the caller's 2428/// incoming argument stack. 2429static 2430bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, 2431 MachineFrameInfo *MFI, const MachineRegisterInfo *MRI, 2432 const X86InstrInfo *TII) { 2433 unsigned Bytes = Arg.getValueType().getSizeInBits() / 8; 2434 int FI = INT_MAX; 2435 if (Arg.getOpcode() == ISD::CopyFromReg) { 2436 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); 2437 if (!TargetRegisterInfo::isVirtualRegister(VR)) 2438 return false; 2439 MachineInstr *Def = MRI->getVRegDef(VR); 2440 if (!Def) 2441 return false; 2442 if (!Flags.isByVal()) { 2443 if (!TII->isLoadFromStackSlot(Def, FI)) 2444 return false; 2445 } else { 2446 unsigned Opcode = Def->getOpcode(); 2447 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r) && 2448 Def->getOperand(1).isFI()) { 2449 FI = Def->getOperand(1).getIndex(); 2450 Bytes = Flags.getByValSize(); 2451 } else 2452 return false; 2453 } 2454 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) { 2455 if (Flags.isByVal()) 2456 // ByVal argument is passed in as a pointer but it's now being 2457 // dereferenced. e.g. 2458 // define @foo(%struct.X* %A) { 2459 // tail call @bar(%struct.X* byval %A) 2460 // } 2461 return false; 2462 SDValue Ptr = Ld->getBasePtr(); 2463 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr); 2464 if (!FINode) 2465 return false; 2466 FI = FINode->getIndex(); 2467 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) { 2468 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg); 2469 FI = FINode->getIndex(); 2470 Bytes = Flags.getByValSize(); 2471 } else 2472 return false; 2473 2474 assert(FI != INT_MAX); 2475 if (!MFI->isFixedObjectIndex(FI)) 2476 return false; 2477 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI); 2478} 2479 2480/// IsEligibleForTailCallOptimization - Check whether the call is eligible 2481/// for tail call optimization. Targets which want to do tail call 2482/// optimization should implement this function. 2483bool 2484X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 2485 CallingConv::ID CalleeCC, 2486 bool isVarArg, 2487 bool isCalleeStructRet, 2488 bool isCallerStructRet, 2489 const SmallVectorImpl<ISD::OutputArg> &Outs, 2490 const SmallVectorImpl<SDValue> &OutVals, 2491 const SmallVectorImpl<ISD::InputArg> &Ins, 2492 SelectionDAG& DAG) const { 2493 if (!IsTailCallConvention(CalleeCC) && 2494 CalleeCC != CallingConv::C) 2495 return false; 2496 2497 // If -tailcallopt is specified, make fastcc functions tail-callable. 2498 const MachineFunction &MF = DAG.getMachineFunction(); 2499 const Function *CallerF = DAG.getMachineFunction().getFunction(); 2500 CallingConv::ID CallerCC = CallerF->getCallingConv(); 2501 bool CCMatch = CallerCC == CalleeCC; 2502 2503 if (GuaranteedTailCallOpt) { 2504 if (IsTailCallConvention(CalleeCC) && CCMatch) 2505 return true; 2506 return false; 2507 } 2508 2509 // Look for obvious safe cases to perform tail call optimization that do not 2510 // require ABI changes. This is what gcc calls sibcall. 2511 2512 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to 2513 // emit a special epilogue. 2514 if (RegInfo->needsStackRealignment(MF)) 2515 return false; 2516 2517 // Also avoid sibcall optimization if either caller or callee uses struct 2518 // return semantics. 2519 if (isCalleeStructRet || isCallerStructRet) 2520 return false; 2521 2522 // An stdcall caller is expected to clean up its arguments; the callee 2523 // isn't going to do that. 2524 if (!CCMatch && CallerCC==CallingConv::X86_StdCall) 2525 return false; 2526 2527 // Do not sibcall optimize vararg calls unless all arguments are passed via 2528 // registers. 2529 if (isVarArg && !Outs.empty()) { 2530 2531 // Optimizing for varargs on Win64 is unlikely to be safe without 2532 // additional testing. 2533 if (Subtarget->isTargetWin64()) 2534 return false; 2535 2536 SmallVector<CCValAssign, 16> ArgLocs; 2537 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), 2538 getTargetMachine(), ArgLocs, *DAG.getContext()); 2539 2540 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 2541 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) 2542 if (!ArgLocs[i].isRegLoc()) 2543 return false; 2544 } 2545 2546 // If the call result is in ST0 / ST1, it needs to be popped off the x87 stack. 2547 // Therefore if it's not used by the call it is not safe to optimize this into 2548 // a sibcall. 2549 bool Unused = false; 2550 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 2551 if (!Ins[i].Used) { 2552 Unused = true; 2553 break; 2554 } 2555 } 2556 if (Unused) { 2557 SmallVector<CCValAssign, 16> RVLocs; 2558 CCState CCInfo(CalleeCC, false, DAG.getMachineFunction(), 2559 getTargetMachine(), RVLocs, *DAG.getContext()); 2560 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 2561 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 2562 CCValAssign &VA = RVLocs[i]; 2563 if (VA.getLocReg() == X86::ST0 || VA.getLocReg() == X86::ST1) 2564 return false; 2565 } 2566 } 2567 2568 // If the calling conventions do not match, then we'd better make sure the 2569 // results are returned in the same way as what the caller expects. 2570 if (!CCMatch) { 2571 SmallVector<CCValAssign, 16> RVLocs1; 2572 CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), 2573 getTargetMachine(), RVLocs1, *DAG.getContext()); 2574 CCInfo1.AnalyzeCallResult(Ins, RetCC_X86); 2575 2576 SmallVector<CCValAssign, 16> RVLocs2; 2577 CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), 2578 getTargetMachine(), RVLocs2, *DAG.getContext()); 2579 CCInfo2.AnalyzeCallResult(Ins, RetCC_X86); 2580 2581 if (RVLocs1.size() != RVLocs2.size()) 2582 return false; 2583 for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) { 2584 if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc()) 2585 return false; 2586 if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo()) 2587 return false; 2588 if (RVLocs1[i].isRegLoc()) { 2589 if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg()) 2590 return false; 2591 } else { 2592 if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset()) 2593 return false; 2594 } 2595 } 2596 } 2597 2598 // If the callee takes no arguments then go on to check the results of the 2599 // call. 2600 if (!Outs.empty()) { 2601 // Check if stack adjustment is needed. For now, do not do this if any 2602 // argument is passed on the stack. 2603 SmallVector<CCValAssign, 16> ArgLocs; 2604 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), 2605 getTargetMachine(), ArgLocs, *DAG.getContext()); 2606 2607 // Allocate shadow area for Win64 2608 if (Subtarget->isTargetWin64()) { 2609 CCInfo.AllocateStack(32, 8); 2610 } 2611 2612 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 2613 if (CCInfo.getNextStackOffset()) { 2614 MachineFunction &MF = DAG.getMachineFunction(); 2615 if (MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn()) 2616 return false; 2617 2618 // Check if the arguments are already laid out in the right way as 2619 // the caller's fixed stack objects. 2620 MachineFrameInfo *MFI = MF.getFrameInfo(); 2621 const MachineRegisterInfo *MRI = &MF.getRegInfo(); 2622 const X86InstrInfo *TII = 2623 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 2624 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2625 CCValAssign &VA = ArgLocs[i]; 2626 SDValue Arg = OutVals[i]; 2627 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2628 if (VA.getLocInfo() == CCValAssign::Indirect) 2629 return false; 2630 if (!VA.isRegLoc()) { 2631 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, 2632 MFI, MRI, TII)) 2633 return false; 2634 } 2635 } 2636 } 2637 2638 // If the tailcall address may be in a register, then make sure it's 2639 // possible to register allocate for it. In 32-bit, the call address can 2640 // only target EAX, EDX, or ECX since the tail call must be scheduled after 2641 // callee-saved registers are restored. These happen to be the same 2642 // registers used to pass 'inreg' arguments so watch out for those. 2643 if (!Subtarget->is64Bit() && 2644 !isa<GlobalAddressSDNode>(Callee) && 2645 !isa<ExternalSymbolSDNode>(Callee)) { 2646 unsigned NumInRegs = 0; 2647 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2648 CCValAssign &VA = ArgLocs[i]; 2649 if (!VA.isRegLoc()) 2650 continue; 2651 unsigned Reg = VA.getLocReg(); 2652 switch (Reg) { 2653 default: break; 2654 case X86::EAX: case X86::EDX: case X86::ECX: 2655 if (++NumInRegs == 3) 2656 return false; 2657 break; 2658 } 2659 } 2660 } 2661 } 2662 2663 return true; 2664} 2665 2666FastISel * 2667X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo) const { 2668 return X86::createFastISel(funcInfo); 2669} 2670 2671 2672//===----------------------------------------------------------------------===// 2673// Other Lowering Hooks 2674//===----------------------------------------------------------------------===// 2675 2676static bool MayFoldLoad(SDValue Op) { 2677 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode()); 2678} 2679 2680static bool MayFoldIntoStore(SDValue Op) { 2681 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); 2682} 2683 2684static bool isTargetShuffle(unsigned Opcode) { 2685 switch(Opcode) { 2686 default: return false; 2687 case X86ISD::PSHUFD: 2688 case X86ISD::PSHUFHW: 2689 case X86ISD::PSHUFLW: 2690 case X86ISD::SHUFPD: 2691 case X86ISD::PALIGN: 2692 case X86ISD::SHUFPS: 2693 case X86ISD::MOVLHPS: 2694 case X86ISD::MOVLHPD: 2695 case X86ISD::MOVHLPS: 2696 case X86ISD::MOVLPS: 2697 case X86ISD::MOVLPD: 2698 case X86ISD::MOVSHDUP: 2699 case X86ISD::MOVSLDUP: 2700 case X86ISD::MOVDDUP: 2701 case X86ISD::MOVSS: 2702 case X86ISD::MOVSD: 2703 case X86ISD::UNPCKLPS: 2704 case X86ISD::UNPCKLPD: 2705 case X86ISD::VUNPCKLPSY: 2706 case X86ISD::VUNPCKLPDY: 2707 case X86ISD::PUNPCKLWD: 2708 case X86ISD::PUNPCKLBW: 2709 case X86ISD::PUNPCKLDQ: 2710 case X86ISD::PUNPCKLQDQ: 2711 case X86ISD::UNPCKHPS: 2712 case X86ISD::UNPCKHPD: 2713 case X86ISD::PUNPCKHWD: 2714 case X86ISD::PUNPCKHBW: 2715 case X86ISD::PUNPCKHDQ: 2716 case X86ISD::PUNPCKHQDQ: 2717 case X86ISD::VPERMIL: 2718 return true; 2719 } 2720 return false; 2721} 2722 2723static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2724 SDValue V1, SelectionDAG &DAG) { 2725 switch(Opc) { 2726 default: llvm_unreachable("Unknown x86 shuffle node"); 2727 case X86ISD::MOVSHDUP: 2728 case X86ISD::MOVSLDUP: 2729 case X86ISD::MOVDDUP: 2730 return DAG.getNode(Opc, dl, VT, V1); 2731 } 2732 2733 return SDValue(); 2734} 2735 2736static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2737 SDValue V1, unsigned TargetMask, SelectionDAG &DAG) { 2738 switch(Opc) { 2739 default: llvm_unreachable("Unknown x86 shuffle node"); 2740 case X86ISD::PSHUFD: 2741 case X86ISD::PSHUFHW: 2742 case X86ISD::PSHUFLW: 2743 case X86ISD::VPERMIL: 2744 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8)); 2745 } 2746 2747 return SDValue(); 2748} 2749 2750static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2751 SDValue V1, SDValue V2, unsigned TargetMask, SelectionDAG &DAG) { 2752 switch(Opc) { 2753 default: llvm_unreachable("Unknown x86 shuffle node"); 2754 case X86ISD::PALIGN: 2755 case X86ISD::SHUFPD: 2756 case X86ISD::SHUFPS: 2757 return DAG.getNode(Opc, dl, VT, V1, V2, 2758 DAG.getConstant(TargetMask, MVT::i8)); 2759 } 2760 return SDValue(); 2761} 2762 2763static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2764 SDValue V1, SDValue V2, SelectionDAG &DAG) { 2765 switch(Opc) { 2766 default: llvm_unreachable("Unknown x86 shuffle node"); 2767 case X86ISD::MOVLHPS: 2768 case X86ISD::MOVLHPD: 2769 case X86ISD::MOVHLPS: 2770 case X86ISD::MOVLPS: 2771 case X86ISD::MOVLPD: 2772 case X86ISD::MOVSS: 2773 case X86ISD::MOVSD: 2774 case X86ISD::UNPCKLPS: 2775 case X86ISD::UNPCKLPD: 2776 case X86ISD::VUNPCKLPSY: 2777 case X86ISD::VUNPCKLPDY: 2778 case X86ISD::PUNPCKLWD: 2779 case X86ISD::PUNPCKLBW: 2780 case X86ISD::PUNPCKLDQ: 2781 case X86ISD::PUNPCKLQDQ: 2782 case X86ISD::UNPCKHPS: 2783 case X86ISD::UNPCKHPD: 2784 case X86ISD::PUNPCKHWD: 2785 case X86ISD::PUNPCKHBW: 2786 case X86ISD::PUNPCKHDQ: 2787 case X86ISD::PUNPCKHQDQ: 2788 return DAG.getNode(Opc, dl, VT, V1, V2); 2789 } 2790 return SDValue(); 2791} 2792 2793SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const { 2794 MachineFunction &MF = DAG.getMachineFunction(); 2795 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 2796 int ReturnAddrIndex = FuncInfo->getRAIndex(); 2797 2798 if (ReturnAddrIndex == 0) { 2799 // Set up a frame object for the return address. 2800 uint64_t SlotSize = TD->getPointerSize(); 2801 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize, 2802 false); 2803 FuncInfo->setRAIndex(ReturnAddrIndex); 2804 } 2805 2806 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 2807} 2808 2809 2810bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, 2811 bool hasSymbolicDisplacement) { 2812 // Offset should fit into 32 bit immediate field. 2813 if (!isInt<32>(Offset)) 2814 return false; 2815 2816 // If we don't have a symbolic displacement - we don't have any extra 2817 // restrictions. 2818 if (!hasSymbolicDisplacement) 2819 return true; 2820 2821 // FIXME: Some tweaks might be needed for medium code model. 2822 if (M != CodeModel::Small && M != CodeModel::Kernel) 2823 return false; 2824 2825 // For small code model we assume that latest object is 16MB before end of 31 2826 // bits boundary. We may also accept pretty large negative constants knowing 2827 // that all objects are in the positive half of address space. 2828 if (M == CodeModel::Small && Offset < 16*1024*1024) 2829 return true; 2830 2831 // For kernel code model we know that all object resist in the negative half 2832 // of 32bits address space. We may not accept negative offsets, since they may 2833 // be just off and we may accept pretty large positive ones. 2834 if (M == CodeModel::Kernel && Offset > 0) 2835 return true; 2836 2837 return false; 2838} 2839 2840/// isCalleePop - Determines whether the callee is required to pop its 2841/// own arguments. Callee pop is necessary to support tail calls. 2842bool X86::isCalleePop(CallingConv::ID CallingConv, 2843 bool is64Bit, bool IsVarArg, bool TailCallOpt) { 2844 if (IsVarArg) 2845 return false; 2846 2847 switch (CallingConv) { 2848 default: 2849 return false; 2850 case CallingConv::X86_StdCall: 2851 return !is64Bit; 2852 case CallingConv::X86_FastCall: 2853 return !is64Bit; 2854 case CallingConv::X86_ThisCall: 2855 return !is64Bit; 2856 case CallingConv::Fast: 2857 return TailCallOpt; 2858 case CallingConv::GHC: 2859 return TailCallOpt; 2860 } 2861} 2862 2863/// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 2864/// specific condition code, returning the condition code and the LHS/RHS of the 2865/// comparison to make. 2866static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 2867 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) { 2868 if (!isFP) { 2869 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 2870 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 2871 // X > -1 -> X == 0, jump !sign. 2872 RHS = DAG.getConstant(0, RHS.getValueType()); 2873 return X86::COND_NS; 2874 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 2875 // X < 0 -> X == 0, jump on sign. 2876 return X86::COND_S; 2877 } else if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) { 2878 // X < 1 -> X <= 0 2879 RHS = DAG.getConstant(0, RHS.getValueType()); 2880 return X86::COND_LE; 2881 } 2882 } 2883 2884 switch (SetCCOpcode) { 2885 default: llvm_unreachable("Invalid integer condition!"); 2886 case ISD::SETEQ: return X86::COND_E; 2887 case ISD::SETGT: return X86::COND_G; 2888 case ISD::SETGE: return X86::COND_GE; 2889 case ISD::SETLT: return X86::COND_L; 2890 case ISD::SETLE: return X86::COND_LE; 2891 case ISD::SETNE: return X86::COND_NE; 2892 case ISD::SETULT: return X86::COND_B; 2893 case ISD::SETUGT: return X86::COND_A; 2894 case ISD::SETULE: return X86::COND_BE; 2895 case ISD::SETUGE: return X86::COND_AE; 2896 } 2897 } 2898 2899 // First determine if it is required or is profitable to flip the operands. 2900 2901 // If LHS is a foldable load, but RHS is not, flip the condition. 2902 if (ISD::isNON_EXTLoad(LHS.getNode()) && 2903 !ISD::isNON_EXTLoad(RHS.getNode())) { 2904 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); 2905 std::swap(LHS, RHS); 2906 } 2907 2908 switch (SetCCOpcode) { 2909 default: break; 2910 case ISD::SETOLT: 2911 case ISD::SETOLE: 2912 case ISD::SETUGT: 2913 case ISD::SETUGE: 2914 std::swap(LHS, RHS); 2915 break; 2916 } 2917 2918 // On a floating point condition, the flags are set as follows: 2919 // ZF PF CF op 2920 // 0 | 0 | 0 | X > Y 2921 // 0 | 0 | 1 | X < Y 2922 // 1 | 0 | 0 | X == Y 2923 // 1 | 1 | 1 | unordered 2924 switch (SetCCOpcode) { 2925 default: llvm_unreachable("Condcode should be pre-legalized away"); 2926 case ISD::SETUEQ: 2927 case ISD::SETEQ: return X86::COND_E; 2928 case ISD::SETOLT: // flipped 2929 case ISD::SETOGT: 2930 case ISD::SETGT: return X86::COND_A; 2931 case ISD::SETOLE: // flipped 2932 case ISD::SETOGE: 2933 case ISD::SETGE: return X86::COND_AE; 2934 case ISD::SETUGT: // flipped 2935 case ISD::SETULT: 2936 case ISD::SETLT: return X86::COND_B; 2937 case ISD::SETUGE: // flipped 2938 case ISD::SETULE: 2939 case ISD::SETLE: return X86::COND_BE; 2940 case ISD::SETONE: 2941 case ISD::SETNE: return X86::COND_NE; 2942 case ISD::SETUO: return X86::COND_P; 2943 case ISD::SETO: return X86::COND_NP; 2944 case ISD::SETOEQ: 2945 case ISD::SETUNE: return X86::COND_INVALID; 2946 } 2947} 2948 2949/// hasFPCMov - is there a floating point cmov for the specific X86 condition 2950/// code. Current x86 isa includes the following FP cmov instructions: 2951/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 2952static bool hasFPCMov(unsigned X86CC) { 2953 switch (X86CC) { 2954 default: 2955 return false; 2956 case X86::COND_B: 2957 case X86::COND_BE: 2958 case X86::COND_E: 2959 case X86::COND_P: 2960 case X86::COND_A: 2961 case X86::COND_AE: 2962 case X86::COND_NE: 2963 case X86::COND_NP: 2964 return true; 2965 } 2966} 2967 2968/// isFPImmLegal - Returns true if the target can instruction select the 2969/// specified FP immediate natively. If false, the legalizer will 2970/// materialize the FP immediate as a load from a constant pool. 2971bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 2972 for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) { 2973 if (Imm.bitwiseIsEqual(LegalFPImmediates[i])) 2974 return true; 2975 } 2976 return false; 2977} 2978 2979/// isUndefOrInRange - Return true if Val is undef or if its value falls within 2980/// the specified range (L, H]. 2981static bool isUndefOrInRange(int Val, int Low, int Hi) { 2982 return (Val < 0) || (Val >= Low && Val < Hi); 2983} 2984 2985/// isUndefOrEqual - Val is either less than zero (undef) or equal to the 2986/// specified value. 2987static bool isUndefOrEqual(int Val, int CmpVal) { 2988 if (Val < 0 || Val == CmpVal) 2989 return true; 2990 return false; 2991} 2992 2993/// isPSHUFDMask - Return true if the node specifies a shuffle of elements that 2994/// is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference 2995/// the second operand. 2996static bool isPSHUFDMask(const SmallVectorImpl<int> &Mask, EVT VT) { 2997 if (VT == MVT::v4f32 || VT == MVT::v4i32 ) 2998 return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4); 2999 if (VT == MVT::v2f64 || VT == MVT::v2i64) 3000 return (Mask[0] < 2 && Mask[1] < 2); 3001 return false; 3002} 3003 3004bool X86::isPSHUFDMask(ShuffleVectorSDNode *N) { 3005 SmallVector<int, 8> M; 3006 N->getMask(M); 3007 return ::isPSHUFDMask(M, N->getValueType(0)); 3008} 3009 3010/// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that 3011/// is suitable for input to PSHUFHW. 3012static bool isPSHUFHWMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3013 if (VT != MVT::v8i16) 3014 return false; 3015 3016 // Lower quadword copied in order or undef. 3017 for (int i = 0; i != 4; ++i) 3018 if (Mask[i] >= 0 && Mask[i] != i) 3019 return false; 3020 3021 // Upper quadword shuffled. 3022 for (int i = 4; i != 8; ++i) 3023 if (Mask[i] >= 0 && (Mask[i] < 4 || Mask[i] > 7)) 3024 return false; 3025 3026 return true; 3027} 3028 3029bool X86::isPSHUFHWMask(ShuffleVectorSDNode *N) { 3030 SmallVector<int, 8> M; 3031 N->getMask(M); 3032 return ::isPSHUFHWMask(M, N->getValueType(0)); 3033} 3034 3035/// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that 3036/// is suitable for input to PSHUFLW. 3037static bool isPSHUFLWMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3038 if (VT != MVT::v8i16) 3039 return false; 3040 3041 // Upper quadword copied in order. 3042 for (int i = 4; i != 8; ++i) 3043 if (Mask[i] >= 0 && Mask[i] != i) 3044 return false; 3045 3046 // Lower quadword shuffled. 3047 for (int i = 0; i != 4; ++i) 3048 if (Mask[i] >= 4) 3049 return false; 3050 3051 return true; 3052} 3053 3054bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) { 3055 SmallVector<int, 8> M; 3056 N->getMask(M); 3057 return ::isPSHUFLWMask(M, N->getValueType(0)); 3058} 3059 3060/// isPALIGNRMask - Return true if the node specifies a shuffle of elements that 3061/// is suitable for input to PALIGNR. 3062static bool isPALIGNRMask(const SmallVectorImpl<int> &Mask, EVT VT, 3063 bool hasSSSE3) { 3064 int i, e = VT.getVectorNumElements(); 3065 3066 // Do not handle v2i64 / v2f64 shuffles with palignr. 3067 if (e < 4 || !hasSSSE3) 3068 return false; 3069 3070 for (i = 0; i != e; ++i) 3071 if (Mask[i] >= 0) 3072 break; 3073 3074 // All undef, not a palignr. 3075 if (i == e) 3076 return false; 3077 3078 // Determine if it's ok to perform a palignr with only the LHS, since we 3079 // don't have access to the actual shuffle elements to see if RHS is undef. 3080 bool Unary = Mask[i] < (int)e; 3081 bool NeedsUnary = false; 3082 3083 int s = Mask[i] - i; 3084 3085 // Check the rest of the elements to see if they are consecutive. 3086 for (++i; i != e; ++i) { 3087 int m = Mask[i]; 3088 if (m < 0) 3089 continue; 3090 3091 Unary = Unary && (m < (int)e); 3092 NeedsUnary = NeedsUnary || (m < s); 3093 3094 if (NeedsUnary && !Unary) 3095 return false; 3096 if (Unary && m != ((s+i) & (e-1))) 3097 return false; 3098 if (!Unary && m != (s+i)) 3099 return false; 3100 } 3101 return true; 3102} 3103 3104bool X86::isPALIGNRMask(ShuffleVectorSDNode *N) { 3105 SmallVector<int, 8> M; 3106 N->getMask(M); 3107 return ::isPALIGNRMask(M, N->getValueType(0), true); 3108} 3109 3110/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 3111/// specifies a shuffle of elements that is suitable for input to SHUFP*. 3112static bool isSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3113 int NumElems = VT.getVectorNumElements(); 3114 if (NumElems != 2 && NumElems != 4) 3115 return false; 3116 3117 int Half = NumElems / 2; 3118 for (int i = 0; i < Half; ++i) 3119 if (!isUndefOrInRange(Mask[i], 0, NumElems)) 3120 return false; 3121 for (int i = Half; i < NumElems; ++i) 3122 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2)) 3123 return false; 3124 3125 return true; 3126} 3127 3128bool X86::isSHUFPMask(ShuffleVectorSDNode *N) { 3129 SmallVector<int, 8> M; 3130 N->getMask(M); 3131 return ::isSHUFPMask(M, N->getValueType(0)); 3132} 3133 3134/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 3135/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 3136/// half elements to come from vector 1 (which would equal the dest.) and 3137/// the upper half to come from vector 2. 3138static bool isCommutedSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3139 int NumElems = VT.getVectorNumElements(); 3140 3141 if (NumElems != 2 && NumElems != 4) 3142 return false; 3143 3144 int Half = NumElems / 2; 3145 for (int i = 0; i < Half; ++i) 3146 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2)) 3147 return false; 3148 for (int i = Half; i < NumElems; ++i) 3149 if (!isUndefOrInRange(Mask[i], 0, NumElems)) 3150 return false; 3151 return true; 3152} 3153 3154static bool isCommutedSHUFP(ShuffleVectorSDNode *N) { 3155 SmallVector<int, 8> M; 3156 N->getMask(M); 3157 return isCommutedSHUFPMask(M, N->getValueType(0)); 3158} 3159 3160/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 3161/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 3162bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) { 3163 if (N->getValueType(0).getVectorNumElements() != 4) 3164 return false; 3165 3166 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 3167 return isUndefOrEqual(N->getMaskElt(0), 6) && 3168 isUndefOrEqual(N->getMaskElt(1), 7) && 3169 isUndefOrEqual(N->getMaskElt(2), 2) && 3170 isUndefOrEqual(N->getMaskElt(3), 3); 3171} 3172 3173/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 3174/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 3175/// <2, 3, 2, 3> 3176bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) { 3177 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 3178 3179 if (NumElems != 4) 3180 return false; 3181 3182 return isUndefOrEqual(N->getMaskElt(0), 2) && 3183 isUndefOrEqual(N->getMaskElt(1), 3) && 3184 isUndefOrEqual(N->getMaskElt(2), 2) && 3185 isUndefOrEqual(N->getMaskElt(3), 3); 3186} 3187 3188/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 3189/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 3190bool X86::isMOVLPMask(ShuffleVectorSDNode *N) { 3191 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 3192 3193 if (NumElems != 2 && NumElems != 4) 3194 return false; 3195 3196 for (unsigned i = 0; i < NumElems/2; ++i) 3197 if (!isUndefOrEqual(N->getMaskElt(i), i + NumElems)) 3198 return false; 3199 3200 for (unsigned i = NumElems/2; i < NumElems; ++i) 3201 if (!isUndefOrEqual(N->getMaskElt(i), i)) 3202 return false; 3203 3204 return true; 3205} 3206 3207/// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand 3208/// specifies a shuffle of elements that is suitable for input to MOVLHPS. 3209bool X86::isMOVLHPSMask(ShuffleVectorSDNode *N) { 3210 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 3211 3212 if ((NumElems != 2 && NumElems != 4) 3213 || N->getValueType(0).getSizeInBits() > 128) 3214 return false; 3215 3216 for (unsigned i = 0; i < NumElems/2; ++i) 3217 if (!isUndefOrEqual(N->getMaskElt(i), i)) 3218 return false; 3219 3220 for (unsigned i = 0; i < NumElems/2; ++i) 3221 if (!isUndefOrEqual(N->getMaskElt(i + NumElems/2), i + NumElems)) 3222 return false; 3223 3224 return true; 3225} 3226 3227/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 3228/// specifies a shuffle of elements that is suitable for input to UNPCKL. 3229static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT, 3230 bool V2IsSplat = false) { 3231 int NumElts = VT.getVectorNumElements(); 3232 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 3233 return false; 3234 3235 // Handle vector lengths > 128 bits. Define a "section" as a set of 3236 // 128 bits. AVX defines UNPCK* to operate independently on 128-bit 3237 // sections. 3238 unsigned NumSections = VT.getSizeInBits() / 128; 3239 if (NumSections == 0 ) NumSections = 1; // Handle MMX 3240 unsigned NumSectionElts = NumElts / NumSections; 3241 3242 unsigned Start = 0; 3243 unsigned End = NumSectionElts; 3244 for (unsigned s = 0; s < NumSections; ++s) { 3245 for (unsigned i = Start, j = s * NumSectionElts; 3246 i != End; 3247 i += 2, ++j) { 3248 int BitI = Mask[i]; 3249 int BitI1 = Mask[i+1]; 3250 if (!isUndefOrEqual(BitI, j)) 3251 return false; 3252 if (V2IsSplat) { 3253 if (!isUndefOrEqual(BitI1, NumElts)) 3254 return false; 3255 } else { 3256 if (!isUndefOrEqual(BitI1, j + NumElts)) 3257 return false; 3258 } 3259 } 3260 // Process the next 128 bits. 3261 Start += NumSectionElts; 3262 End += NumSectionElts; 3263 } 3264 3265 return true; 3266} 3267 3268bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) { 3269 SmallVector<int, 8> M; 3270 N->getMask(M); 3271 return ::isUNPCKLMask(M, N->getValueType(0), V2IsSplat); 3272} 3273 3274/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 3275/// specifies a shuffle of elements that is suitable for input to UNPCKH. 3276static bool isUNPCKHMask(const SmallVectorImpl<int> &Mask, EVT VT, 3277 bool V2IsSplat = false) { 3278 int NumElts = VT.getVectorNumElements(); 3279 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 3280 return false; 3281 3282 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) { 3283 int BitI = Mask[i]; 3284 int BitI1 = Mask[i+1]; 3285 if (!isUndefOrEqual(BitI, j + NumElts/2)) 3286 return false; 3287 if (V2IsSplat) { 3288 if (isUndefOrEqual(BitI1, NumElts)) 3289 return false; 3290 } else { 3291 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 3292 return false; 3293 } 3294 } 3295 return true; 3296} 3297 3298bool X86::isUNPCKHMask(ShuffleVectorSDNode *N, bool V2IsSplat) { 3299 SmallVector<int, 8> M; 3300 N->getMask(M); 3301 return ::isUNPCKHMask(M, N->getValueType(0), V2IsSplat); 3302} 3303 3304/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 3305/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 3306/// <0, 0, 1, 1> 3307static bool isUNPCKL_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) { 3308 int NumElems = VT.getVectorNumElements(); 3309 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 3310 return false; 3311 3312 // Handle vector lengths > 128 bits. Define a "section" as a set of 3313 // 128 bits. AVX defines UNPCK* to operate independently on 128-bit 3314 // sections. 3315 unsigned NumSections = VT.getSizeInBits() / 128; 3316 if (NumSections == 0 ) NumSections = 1; // Handle MMX 3317 unsigned NumSectionElts = NumElems / NumSections; 3318 3319 for (unsigned s = 0; s < NumSections; ++s) { 3320 for (unsigned i = s * NumSectionElts, j = s * NumSectionElts; 3321 i != NumSectionElts * (s + 1); 3322 i += 2, ++j) { 3323 int BitI = Mask[i]; 3324 int BitI1 = Mask[i+1]; 3325 3326 if (!isUndefOrEqual(BitI, j)) 3327 return false; 3328 if (!isUndefOrEqual(BitI1, j)) 3329 return false; 3330 } 3331 } 3332 3333 return true; 3334} 3335 3336bool X86::isUNPCKL_v_undef_Mask(ShuffleVectorSDNode *N) { 3337 SmallVector<int, 8> M; 3338 N->getMask(M); 3339 return ::isUNPCKL_v_undef_Mask(M, N->getValueType(0)); 3340} 3341 3342/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 3343/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 3344/// <2, 2, 3, 3> 3345static bool isUNPCKH_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) { 3346 int NumElems = VT.getVectorNumElements(); 3347 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 3348 return false; 3349 3350 for (int i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 3351 int BitI = Mask[i]; 3352 int BitI1 = Mask[i+1]; 3353 if (!isUndefOrEqual(BitI, j)) 3354 return false; 3355 if (!isUndefOrEqual(BitI1, j)) 3356 return false; 3357 } 3358 return true; 3359} 3360 3361bool X86::isUNPCKH_v_undef_Mask(ShuffleVectorSDNode *N) { 3362 SmallVector<int, 8> M; 3363 N->getMask(M); 3364 return ::isUNPCKH_v_undef_Mask(M, N->getValueType(0)); 3365} 3366 3367/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 3368/// specifies a shuffle of elements that is suitable for input to MOVSS, 3369/// MOVSD, and MOVD, i.e. setting the lowest element. 3370static bool isMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3371 if (VT.getVectorElementType().getSizeInBits() < 32) 3372 return false; 3373 3374 int NumElts = VT.getVectorNumElements(); 3375 3376 if (!isUndefOrEqual(Mask[0], NumElts)) 3377 return false; 3378 3379 for (int i = 1; i < NumElts; ++i) 3380 if (!isUndefOrEqual(Mask[i], i)) 3381 return false; 3382 3383 return true; 3384} 3385 3386bool X86::isMOVLMask(ShuffleVectorSDNode *N) { 3387 SmallVector<int, 8> M; 3388 N->getMask(M); 3389 return ::isMOVLMask(M, N->getValueType(0)); 3390} 3391 3392/// isVPERMILMask - Return true if the specified VECTOR_SHUFFLE operand 3393/// specifies a shuffle of elements that is suitable for input to VPERMIL*. 3394static bool isVPERMILMask(const SmallVectorImpl<int> &Mask, EVT VT) { 3395 unsigned NumElts = VT.getVectorNumElements(); 3396 unsigned NumLanes = VT.getSizeInBits()/128; 3397 3398 // Match any permutation of 128-bit vector with 32/64-bit types 3399 if (NumLanes == 1) { 3400 if (NumElts == 4 || NumElts == 2) 3401 return true; 3402 return false; 3403 } 3404 3405 // Only match 256-bit with 32/64-bit types 3406 if (NumElts != 8 && NumElts != 4) 3407 return false; 3408 3409 // The mask on the high lane should be the same as the low. Actually, 3410 // they can differ if any of the corresponding index in a lane is undef. 3411 int LaneSize = NumElts/NumLanes; 3412 for (int i = 0; i < LaneSize; ++i) { 3413 int HighElt = i+LaneSize; 3414 if (Mask[i] < 0 || Mask[HighElt] < 0) 3415 continue; 3416 3417 if (Mask[HighElt]-Mask[i] != LaneSize) 3418 return false; 3419 } 3420 3421 return true; 3422} 3423 3424/// getShuffleVPERMILImmediateediate - Return the appropriate immediate to shuffle 3425/// the specified VECTOR_MASK mask with VPERMIL* instructions. 3426static unsigned getShuffleVPERMILImmediate(SDNode *N) { 3427 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 3428 EVT VT = SVOp->getValueType(0); 3429 3430 int NumElts = VT.getVectorNumElements(); 3431 int NumLanes = VT.getSizeInBits()/128; 3432 3433 unsigned Mask = 0; 3434 for (int i = 0; i < NumElts/NumLanes /* lane size */; ++i) 3435 Mask |= SVOp->getMaskElt(i) << (i*2); 3436 3437 return Mask; 3438} 3439 3440/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 3441/// of what x86 movss want. X86 movs requires the lowest element to be lowest 3442/// element of vector 2 and the other elements to come from vector 1 in order. 3443static bool isCommutedMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT, 3444 bool V2IsSplat = false, bool V2IsUndef = false) { 3445 int NumOps = VT.getVectorNumElements(); 3446 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 3447 return false; 3448 3449 if (!isUndefOrEqual(Mask[0], 0)) 3450 return false; 3451 3452 for (int i = 1; i < NumOps; ++i) 3453 if (!(isUndefOrEqual(Mask[i], i+NumOps) || 3454 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) || 3455 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps)))) 3456 return false; 3457 3458 return true; 3459} 3460 3461static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false, 3462 bool V2IsUndef = false) { 3463 SmallVector<int, 8> M; 3464 N->getMask(M); 3465 return isCommutedMOVLMask(M, N->getValueType(0), V2IsSplat, V2IsUndef); 3466} 3467 3468/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3469/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 3470bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) { 3471 if (N->getValueType(0).getVectorNumElements() != 4) 3472 return false; 3473 3474 // Expect 1, 1, 3, 3 3475 for (unsigned i = 0; i < 2; ++i) { 3476 int Elt = N->getMaskElt(i); 3477 if (Elt >= 0 && Elt != 1) 3478 return false; 3479 } 3480 3481 bool HasHi = false; 3482 for (unsigned i = 2; i < 4; ++i) { 3483 int Elt = N->getMaskElt(i); 3484 if (Elt >= 0 && Elt != 3) 3485 return false; 3486 if (Elt == 3) 3487 HasHi = true; 3488 } 3489 // Don't use movshdup if it can be done with a shufps. 3490 // FIXME: verify that matching u, u, 3, 3 is what we want. 3491 return HasHi; 3492} 3493 3494/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3495/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 3496bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) { 3497 if (N->getValueType(0).getVectorNumElements() != 4) 3498 return false; 3499 3500 // Expect 0, 0, 2, 2 3501 for (unsigned i = 0; i < 2; ++i) 3502 if (N->getMaskElt(i) > 0) 3503 return false; 3504 3505 bool HasHi = false; 3506 for (unsigned i = 2; i < 4; ++i) { 3507 int Elt = N->getMaskElt(i); 3508 if (Elt >= 0 && Elt != 2) 3509 return false; 3510 if (Elt == 2) 3511 HasHi = true; 3512 } 3513 // Don't use movsldup if it can be done with a shufps. 3514 return HasHi; 3515} 3516 3517/// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3518/// specifies a shuffle of elements that is suitable for input to MOVDDUP. 3519bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) { 3520 int e = N->getValueType(0).getVectorNumElements() / 2; 3521 3522 for (int i = 0; i < e; ++i) 3523 if (!isUndefOrEqual(N->getMaskElt(i), i)) 3524 return false; 3525 for (int i = 0; i < e; ++i) 3526 if (!isUndefOrEqual(N->getMaskElt(e+i), i)) 3527 return false; 3528 return true; 3529} 3530 3531/// isVEXTRACTF128Index - Return true if the specified 3532/// EXTRACT_SUBVECTOR operand specifies a vector extract that is 3533/// suitable for input to VEXTRACTF128. 3534bool X86::isVEXTRACTF128Index(SDNode *N) { 3535 if (!isa<ConstantSDNode>(N->getOperand(1).getNode())) 3536 return false; 3537 3538 // The index should be aligned on a 128-bit boundary. 3539 uint64_t Index = 3540 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue(); 3541 3542 unsigned VL = N->getValueType(0).getVectorNumElements(); 3543 unsigned VBits = N->getValueType(0).getSizeInBits(); 3544 unsigned ElSize = VBits / VL; 3545 bool Result = (Index * ElSize) % 128 == 0; 3546 3547 return Result; 3548} 3549 3550/// isVINSERTF128Index - Return true if the specified INSERT_SUBVECTOR 3551/// operand specifies a subvector insert that is suitable for input to 3552/// VINSERTF128. 3553bool X86::isVINSERTF128Index(SDNode *N) { 3554 if (!isa<ConstantSDNode>(N->getOperand(2).getNode())) 3555 return false; 3556 3557 // The index should be aligned on a 128-bit boundary. 3558 uint64_t Index = 3559 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue(); 3560 3561 unsigned VL = N->getValueType(0).getVectorNumElements(); 3562 unsigned VBits = N->getValueType(0).getSizeInBits(); 3563 unsigned ElSize = VBits / VL; 3564 bool Result = (Index * ElSize) % 128 == 0; 3565 3566 return Result; 3567} 3568 3569/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 3570/// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions. 3571unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 3572 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 3573 int NumOperands = SVOp->getValueType(0).getVectorNumElements(); 3574 3575 unsigned Shift = (NumOperands == 4) ? 2 : 1; 3576 unsigned Mask = 0; 3577 for (int i = 0; i < NumOperands; ++i) { 3578 int Val = SVOp->getMaskElt(NumOperands-i-1); 3579 if (Val < 0) Val = 0; 3580 if (Val >= NumOperands) Val -= NumOperands; 3581 Mask |= Val; 3582 if (i != NumOperands - 1) 3583 Mask <<= Shift; 3584 } 3585 return Mask; 3586} 3587 3588/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 3589/// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction. 3590unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 3591 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 3592 unsigned Mask = 0; 3593 // 8 nodes, but we only care about the last 4. 3594 for (unsigned i = 7; i >= 4; --i) { 3595 int Val = SVOp->getMaskElt(i); 3596 if (Val >= 0) 3597 Mask |= (Val - 4); 3598 if (i != 4) 3599 Mask <<= 2; 3600 } 3601 return Mask; 3602} 3603 3604/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 3605/// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction. 3606unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 3607 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 3608 unsigned Mask = 0; 3609 // 8 nodes, but we only care about the first 4. 3610 for (int i = 3; i >= 0; --i) { 3611 int Val = SVOp->getMaskElt(i); 3612 if (Val >= 0) 3613 Mask |= Val; 3614 if (i != 0) 3615 Mask <<= 2; 3616 } 3617 return Mask; 3618} 3619 3620/// getShufflePALIGNRImmediate - Return the appropriate immediate to shuffle 3621/// the specified VECTOR_SHUFFLE mask with the PALIGNR instruction. 3622unsigned X86::getShufflePALIGNRImmediate(SDNode *N) { 3623 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 3624 EVT VVT = N->getValueType(0); 3625 unsigned EltSize = VVT.getVectorElementType().getSizeInBits() >> 3; 3626 int Val = 0; 3627 3628 unsigned i, e; 3629 for (i = 0, e = VVT.getVectorNumElements(); i != e; ++i) { 3630 Val = SVOp->getMaskElt(i); 3631 if (Val >= 0) 3632 break; 3633 } 3634 return (Val - i) * EltSize; 3635} 3636 3637/// getExtractVEXTRACTF128Immediate - Return the appropriate immediate 3638/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128 3639/// instructions. 3640unsigned X86::getExtractVEXTRACTF128Immediate(SDNode *N) { 3641 if (!isa<ConstantSDNode>(N->getOperand(1).getNode())) 3642 llvm_unreachable("Illegal extract subvector for VEXTRACTF128"); 3643 3644 uint64_t Index = 3645 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue(); 3646 3647 EVT VecVT = N->getOperand(0).getValueType(); 3648 EVT ElVT = VecVT.getVectorElementType(); 3649 3650 unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits(); 3651 return Index / NumElemsPerChunk; 3652} 3653 3654/// getInsertVINSERTF128Immediate - Return the appropriate immediate 3655/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128 3656/// instructions. 3657unsigned X86::getInsertVINSERTF128Immediate(SDNode *N) { 3658 if (!isa<ConstantSDNode>(N->getOperand(2).getNode())) 3659 llvm_unreachable("Illegal insert subvector for VINSERTF128"); 3660 3661 uint64_t Index = 3662 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue(); 3663 3664 EVT VecVT = N->getValueType(0); 3665 EVT ElVT = VecVT.getVectorElementType(); 3666 3667 unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits(); 3668 return Index / NumElemsPerChunk; 3669} 3670 3671/// isZeroNode - Returns true if Elt is a constant zero or a floating point 3672/// constant +0.0. 3673bool X86::isZeroNode(SDValue Elt) { 3674 return ((isa<ConstantSDNode>(Elt) && 3675 cast<ConstantSDNode>(Elt)->isNullValue()) || 3676 (isa<ConstantFPSDNode>(Elt) && 3677 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 3678} 3679 3680/// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in 3681/// their permute mask. 3682static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp, 3683 SelectionDAG &DAG) { 3684 EVT VT = SVOp->getValueType(0); 3685 unsigned NumElems = VT.getVectorNumElements(); 3686 SmallVector<int, 8> MaskVec; 3687 3688 for (unsigned i = 0; i != NumElems; ++i) { 3689 int idx = SVOp->getMaskElt(i); 3690 if (idx < 0) 3691 MaskVec.push_back(idx); 3692 else if (idx < (int)NumElems) 3693 MaskVec.push_back(idx + NumElems); 3694 else 3695 MaskVec.push_back(idx - NumElems); 3696 } 3697 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1), 3698 SVOp->getOperand(0), &MaskVec[0]); 3699} 3700 3701/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 3702/// the two vector operands have swapped position. 3703static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, EVT VT) { 3704 unsigned NumElems = VT.getVectorNumElements(); 3705 for (unsigned i = 0; i != NumElems; ++i) { 3706 int idx = Mask[i]; 3707 if (idx < 0) 3708 continue; 3709 else if (idx < (int)NumElems) 3710 Mask[i] = idx + NumElems; 3711 else 3712 Mask[i] = idx - NumElems; 3713 } 3714} 3715 3716/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 3717/// match movhlps. The lower half elements should come from upper half of 3718/// V1 (and in order), and the upper half elements should come from the upper 3719/// half of V2 (and in order). 3720static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) { 3721 if (Op->getValueType(0).getVectorNumElements() != 4) 3722 return false; 3723 for (unsigned i = 0, e = 2; i != e; ++i) 3724 if (!isUndefOrEqual(Op->getMaskElt(i), i+2)) 3725 return false; 3726 for (unsigned i = 2; i != 4; ++i) 3727 if (!isUndefOrEqual(Op->getMaskElt(i), i+4)) 3728 return false; 3729 return true; 3730} 3731 3732/// isScalarLoadToVector - Returns true if the node is a scalar load that 3733/// is promoted to a vector. It also returns the LoadSDNode by reference if 3734/// required. 3735static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 3736 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR) 3737 return false; 3738 N = N->getOperand(0).getNode(); 3739 if (!ISD::isNON_EXTLoad(N)) 3740 return false; 3741 if (LD) 3742 *LD = cast<LoadSDNode>(N); 3743 return true; 3744} 3745 3746/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 3747/// match movlp{s|d}. The lower half elements should come from lower half of 3748/// V1 (and in order), and the upper half elements should come from the upper 3749/// half of V2 (and in order). And since V1 will become the source of the 3750/// MOVLP, it must be either a vector load or a scalar load to vector. 3751static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, 3752 ShuffleVectorSDNode *Op) { 3753 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 3754 return false; 3755 // Is V2 is a vector load, don't do this transformation. We will try to use 3756 // load folding shufps op. 3757 if (ISD::isNON_EXTLoad(V2)) 3758 return false; 3759 3760 unsigned NumElems = Op->getValueType(0).getVectorNumElements(); 3761 3762 if (NumElems != 2 && NumElems != 4) 3763 return false; 3764 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 3765 if (!isUndefOrEqual(Op->getMaskElt(i), i)) 3766 return false; 3767 for (unsigned i = NumElems/2; i != NumElems; ++i) 3768 if (!isUndefOrEqual(Op->getMaskElt(i), i+NumElems)) 3769 return false; 3770 return true; 3771} 3772 3773/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 3774/// all the same. 3775static bool isSplatVector(SDNode *N) { 3776 if (N->getOpcode() != ISD::BUILD_VECTOR) 3777 return false; 3778 3779 SDValue SplatValue = N->getOperand(0); 3780 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 3781 if (N->getOperand(i) != SplatValue) 3782 return false; 3783 return true; 3784} 3785 3786/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 3787/// to an zero vector. 3788/// FIXME: move to dag combiner / method on ShuffleVectorSDNode 3789static bool isZeroShuffle(ShuffleVectorSDNode *N) { 3790 SDValue V1 = N->getOperand(0); 3791 SDValue V2 = N->getOperand(1); 3792 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 3793 for (unsigned i = 0; i != NumElems; ++i) { 3794 int Idx = N->getMaskElt(i); 3795 if (Idx >= (int)NumElems) { 3796 unsigned Opc = V2.getOpcode(); 3797 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) 3798 continue; 3799 if (Opc != ISD::BUILD_VECTOR || 3800 !X86::isZeroNode(V2.getOperand(Idx-NumElems))) 3801 return false; 3802 } else if (Idx >= 0) { 3803 unsigned Opc = V1.getOpcode(); 3804 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) 3805 continue; 3806 if (Opc != ISD::BUILD_VECTOR || 3807 !X86::isZeroNode(V1.getOperand(Idx))) 3808 return false; 3809 } 3810 } 3811 return true; 3812} 3813 3814/// getZeroVector - Returns a vector of specified type with all zero elements. 3815/// 3816static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG, 3817 DebugLoc dl) { 3818 assert(VT.isVector() && "Expected a vector type"); 3819 3820 // Always build SSE zero vectors as <4 x i32> bitcasted 3821 // to their dest type. This ensures they get CSE'd. 3822 SDValue Vec; 3823 if (VT.getSizeInBits() == 128) { // SSE 3824 if (HasSSE2) { // SSE2 3825 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 3826 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 3827 } else { // SSE1 3828 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 3829 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); 3830 } 3831 } else if (VT.getSizeInBits() == 256) { // AVX 3832 // 256-bit logic and arithmetic instructions in AVX are 3833 // all floating-point, no support for integer ops. Default 3834 // to emitting fp zeroed vectors then. 3835 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 3836 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; 3837 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops, 8); 3838 } 3839 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); 3840} 3841 3842/// getOnesVector - Returns a vector of specified type with all bits set. 3843/// Always build ones vectors as <4 x i32> or <8 x i32> bitcasted to 3844/// their original type, ensuring they get CSE'd. 3845static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) { 3846 assert(VT.isVector() && "Expected a vector type"); 3847 assert((VT.is128BitVector() || VT.is256BitVector()) 3848 && "Expected a 128-bit or 256-bit vector type"); 3849 3850 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 3851 3852 SDValue Vec; 3853 if (VT.is256BitVector()) { 3854 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; 3855 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops, 8); 3856 } else 3857 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 3858 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); 3859} 3860 3861/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 3862/// that point to V2 points to its first element. 3863static SDValue NormalizeMask(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 3864 EVT VT = SVOp->getValueType(0); 3865 unsigned NumElems = VT.getVectorNumElements(); 3866 3867 bool Changed = false; 3868 SmallVector<int, 8> MaskVec; 3869 SVOp->getMask(MaskVec); 3870 3871 for (unsigned i = 0; i != NumElems; ++i) { 3872 if (MaskVec[i] > (int)NumElems) { 3873 MaskVec[i] = NumElems; 3874 Changed = true; 3875 } 3876 } 3877 if (Changed) 3878 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(0), 3879 SVOp->getOperand(1), &MaskVec[0]); 3880 return SDValue(SVOp, 0); 3881} 3882 3883/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 3884/// operation of specified width. 3885static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 3886 SDValue V2) { 3887 unsigned NumElems = VT.getVectorNumElements(); 3888 SmallVector<int, 8> Mask; 3889 Mask.push_back(NumElems); 3890 for (unsigned i = 1; i != NumElems; ++i) 3891 Mask.push_back(i); 3892 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 3893} 3894 3895/// getUnpackl - Returns a vector_shuffle node for an unpackl operation. 3896static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 3897 SDValue V2) { 3898 unsigned NumElems = VT.getVectorNumElements(); 3899 SmallVector<int, 8> Mask; 3900 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 3901 Mask.push_back(i); 3902 Mask.push_back(i + NumElems); 3903 } 3904 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 3905} 3906 3907/// getUnpackh - Returns a vector_shuffle node for an unpackh operation. 3908static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 3909 SDValue V2) { 3910 unsigned NumElems = VT.getVectorNumElements(); 3911 unsigned Half = NumElems/2; 3912 SmallVector<int, 8> Mask; 3913 for (unsigned i = 0; i != Half; ++i) { 3914 Mask.push_back(i + Half); 3915 Mask.push_back(i + NumElems + Half); 3916 } 3917 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 3918} 3919 3920// PromoteSplatv8v16 - All i16 and i8 vector types can't be used directly by 3921// a generic shuffle instruction because the target has no such instructions. 3922// Generate shuffles which repeat i16 and i8 several times until they can be 3923// represented by v4f32 and then be manipulated by target suported shuffles. 3924static SDValue PromoteSplatv8v16(SDValue V, SelectionDAG &DAG, int &EltNo) { 3925 EVT VT = V.getValueType(); 3926 int NumElems = VT.getVectorNumElements(); 3927 DebugLoc dl = V.getDebugLoc(); 3928 3929 while (NumElems > 4) { 3930 if (EltNo < NumElems/2) { 3931 V = getUnpackl(DAG, dl, VT, V, V); 3932 } else { 3933 V = getUnpackh(DAG, dl, VT, V, V); 3934 EltNo -= NumElems/2; 3935 } 3936 NumElems >>= 1; 3937 } 3938 return V; 3939} 3940 3941/// getLegalSplat - Generate a legal splat with supported x86 shuffles 3942static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) { 3943 EVT VT = V.getValueType(); 3944 DebugLoc dl = V.getDebugLoc(); 3945 assert((VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256) 3946 && "Vector size not supported"); 3947 3948 bool Is128 = VT.getSizeInBits() == 128; 3949 EVT NVT = Is128 ? MVT::v4f32 : MVT::v8f32; 3950 V = DAG.getNode(ISD::BITCAST, dl, NVT, V); 3951 3952 if (Is128) { 3953 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo }; 3954 V = DAG.getVectorShuffle(NVT, dl, V, DAG.getUNDEF(NVT), &SplatMask[0]); 3955 } else { 3956 // The second half of indicies refer to the higher part, which is a 3957 // duplication of the lower one. This makes this shuffle a perfect match 3958 // for the VPERM instruction. 3959 int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo, 3960 EltNo+4, EltNo+4, EltNo+4, EltNo+4 }; 3961 V = DAG.getVectorShuffle(NVT, dl, V, DAG.getUNDEF(NVT), &SplatMask[0]); 3962 } 3963 3964 return DAG.getNode(ISD::BITCAST, dl, VT, V); 3965} 3966 3967/// PromoteSplat - Promote a splat of v4i32, v8i16 or v16i8 to v4f32 and 3968/// v8i32, v16i16 or v32i8 to v8f32. 3969static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) { 3970 EVT SrcVT = SV->getValueType(0); 3971 SDValue V1 = SV->getOperand(0); 3972 DebugLoc dl = SV->getDebugLoc(); 3973 3974 int EltNo = SV->getSplatIndex(); 3975 int NumElems = SrcVT.getVectorNumElements(); 3976 unsigned Size = SrcVT.getSizeInBits(); 3977 3978 // Extract the 128-bit part containing the splat element and update 3979 // the splat element index when it refers to the higher register. 3980 if (Size == 256) { 3981 unsigned Idx = (EltNo > NumElems/2) ? NumElems/2 : 0; 3982 V1 = Extract128BitVector(V1, DAG.getConstant(Idx, MVT::i32), DAG, dl); 3983 if (Idx > 0) 3984 EltNo -= NumElems/2; 3985 } 3986 3987 // Make this 128-bit vector duplicate i8 and i16 elements 3988 if (NumElems > 4) 3989 V1 = PromoteSplatv8v16(V1, DAG, EltNo); 3990 3991 // Recreate the 256-bit vector and place the same 128-bit vector 3992 // into the low and high part. This is necessary because we want 3993 // to use VPERM to shuffle the v8f32 vector, and VPERM only shuffles 3994 // inside each separate v4f32 lane. 3995 if (Size == 256) { 3996 SDValue InsV = Insert128BitVector(DAG.getUNDEF(SrcVT), V1, 3997 DAG.getConstant(0, MVT::i32), DAG, dl); 3998 V1 = Insert128BitVector(InsV, V1, 3999 DAG.getConstant(NumElems/2, MVT::i32), DAG, dl); 4000 } 4001 4002 return getLegalSplat(DAG, V1, EltNo); 4003} 4004 4005/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 4006/// vector of zero or undef vector. This produces a shuffle where the low 4007/// element of V2 is swizzled into the zero/undef vector, landing at element 4008/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 4009static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 4010 bool isZero, bool HasSSE2, 4011 SelectionDAG &DAG) { 4012 EVT VT = V2.getValueType(); 4013 SDValue V1 = isZero 4014 ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT); 4015 unsigned NumElems = VT.getVectorNumElements(); 4016 SmallVector<int, 16> MaskVec; 4017 for (unsigned i = 0; i != NumElems; ++i) 4018 // If this is the insertion idx, put the low elt of V2 here. 4019 MaskVec.push_back(i == Idx ? NumElems : i); 4020 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]); 4021} 4022 4023/// getShuffleScalarElt - Returns the scalar element that will make up the ith 4024/// element of the result of the vector shuffle. 4025static SDValue getShuffleScalarElt(SDNode *N, int Index, SelectionDAG &DAG, 4026 unsigned Depth) { 4027 if (Depth == 6) 4028 return SDValue(); // Limit search depth. 4029 4030 SDValue V = SDValue(N, 0); 4031 EVT VT = V.getValueType(); 4032 unsigned Opcode = V.getOpcode(); 4033 4034 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars. 4035 if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) { 4036 Index = SV->getMaskElt(Index); 4037 4038 if (Index < 0) 4039 return DAG.getUNDEF(VT.getVectorElementType()); 4040 4041 int NumElems = VT.getVectorNumElements(); 4042 SDValue NewV = (Index < NumElems) ? SV->getOperand(0) : SV->getOperand(1); 4043 return getShuffleScalarElt(NewV.getNode(), Index % NumElems, DAG, Depth+1); 4044 } 4045 4046 // Recurse into target specific vector shuffles to find scalars. 4047 if (isTargetShuffle(Opcode)) { 4048 int NumElems = VT.getVectorNumElements(); 4049 SmallVector<unsigned, 16> ShuffleMask; 4050 SDValue ImmN; 4051 4052 switch(Opcode) { 4053 case X86ISD::SHUFPS: 4054 case X86ISD::SHUFPD: 4055 ImmN = N->getOperand(N->getNumOperands()-1); 4056 DecodeSHUFPSMask(NumElems, 4057 cast<ConstantSDNode>(ImmN)->getZExtValue(), 4058 ShuffleMask); 4059 break; 4060 case X86ISD::PUNPCKHBW: 4061 case X86ISD::PUNPCKHWD: 4062 case X86ISD::PUNPCKHDQ: 4063 case X86ISD::PUNPCKHQDQ: 4064 DecodePUNPCKHMask(NumElems, ShuffleMask); 4065 break; 4066 case X86ISD::UNPCKHPS: 4067 case X86ISD::UNPCKHPD: 4068 DecodeUNPCKHPMask(NumElems, ShuffleMask); 4069 break; 4070 case X86ISD::PUNPCKLBW: 4071 case X86ISD::PUNPCKLWD: 4072 case X86ISD::PUNPCKLDQ: 4073 case X86ISD::PUNPCKLQDQ: 4074 DecodePUNPCKLMask(VT, ShuffleMask); 4075 break; 4076 case X86ISD::UNPCKLPS: 4077 case X86ISD::UNPCKLPD: 4078 case X86ISD::VUNPCKLPSY: 4079 case X86ISD::VUNPCKLPDY: 4080 DecodeUNPCKLPMask(VT, ShuffleMask); 4081 break; 4082 case X86ISD::MOVHLPS: 4083 DecodeMOVHLPSMask(NumElems, ShuffleMask); 4084 break; 4085 case X86ISD::MOVLHPS: 4086 DecodeMOVLHPSMask(NumElems, ShuffleMask); 4087 break; 4088 case X86ISD::PSHUFD: 4089 ImmN = N->getOperand(N->getNumOperands()-1); 4090 DecodePSHUFMask(NumElems, 4091 cast<ConstantSDNode>(ImmN)->getZExtValue(), 4092 ShuffleMask); 4093 break; 4094 case X86ISD::PSHUFHW: 4095 ImmN = N->getOperand(N->getNumOperands()-1); 4096 DecodePSHUFHWMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), 4097 ShuffleMask); 4098 break; 4099 case X86ISD::PSHUFLW: 4100 ImmN = N->getOperand(N->getNumOperands()-1); 4101 DecodePSHUFLWMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), 4102 ShuffleMask); 4103 break; 4104 case X86ISD::MOVSS: 4105 case X86ISD::MOVSD: { 4106 // The index 0 always comes from the first element of the second source, 4107 // this is why MOVSS and MOVSD are used in the first place. The other 4108 // elements come from the other positions of the first source vector. 4109 unsigned OpNum = (Index == 0) ? 1 : 0; 4110 return getShuffleScalarElt(V.getOperand(OpNum).getNode(), Index, DAG, 4111 Depth+1); 4112 } 4113 case X86ISD::VPERMIL: 4114 ImmN = N->getOperand(N->getNumOperands()-1); 4115 DecodeVPERMILMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), 4116 ShuffleMask); 4117 default: 4118 assert("not implemented for target shuffle node"); 4119 return SDValue(); 4120 } 4121 4122 Index = ShuffleMask[Index]; 4123 if (Index < 0) 4124 return DAG.getUNDEF(VT.getVectorElementType()); 4125 4126 SDValue NewV = (Index < NumElems) ? N->getOperand(0) : N->getOperand(1); 4127 return getShuffleScalarElt(NewV.getNode(), Index % NumElems, DAG, 4128 Depth+1); 4129 } 4130 4131 // Actual nodes that may contain scalar elements 4132 if (Opcode == ISD::BITCAST) { 4133 V = V.getOperand(0); 4134 EVT SrcVT = V.getValueType(); 4135 unsigned NumElems = VT.getVectorNumElements(); 4136 4137 if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems) 4138 return SDValue(); 4139 } 4140 4141 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) 4142 return (Index == 0) ? V.getOperand(0) 4143 : DAG.getUNDEF(VT.getVectorElementType()); 4144 4145 if (V.getOpcode() == ISD::BUILD_VECTOR) 4146 return V.getOperand(Index); 4147 4148 return SDValue(); 4149} 4150 4151/// getNumOfConsecutiveZeros - Return the number of elements of a vector 4152/// shuffle operation which come from a consecutively from a zero. The 4153/// search can start in two different directions, from left or right. 4154static 4155unsigned getNumOfConsecutiveZeros(SDNode *N, int NumElems, 4156 bool ZerosFromLeft, SelectionDAG &DAG) { 4157 int i = 0; 4158 4159 while (i < NumElems) { 4160 unsigned Index = ZerosFromLeft ? i : NumElems-i-1; 4161 SDValue Elt = getShuffleScalarElt(N, Index, DAG, 0); 4162 if (!(Elt.getNode() && 4163 (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt)))) 4164 break; 4165 ++i; 4166 } 4167 4168 return i; 4169} 4170 4171/// isShuffleMaskConsecutive - Check if the shuffle mask indicies from MaskI to 4172/// MaskE correspond consecutively to elements from one of the vector operands, 4173/// starting from its index OpIdx. Also tell OpNum which source vector operand. 4174static 4175bool isShuffleMaskConsecutive(ShuffleVectorSDNode *SVOp, int MaskI, int MaskE, 4176 int OpIdx, int NumElems, unsigned &OpNum) { 4177 bool SeenV1 = false; 4178 bool SeenV2 = false; 4179 4180 for (int i = MaskI; i <= MaskE; ++i, ++OpIdx) { 4181 int Idx = SVOp->getMaskElt(i); 4182 // Ignore undef indicies 4183 if (Idx < 0) 4184 continue; 4185 4186 if (Idx < NumElems) 4187 SeenV1 = true; 4188 else 4189 SeenV2 = true; 4190 4191 // Only accept consecutive elements from the same vector 4192 if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2)) 4193 return false; 4194 } 4195 4196 OpNum = SeenV1 ? 0 : 1; 4197 return true; 4198} 4199 4200/// isVectorShiftRight - Returns true if the shuffle can be implemented as a 4201/// logical left shift of a vector. 4202static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4203 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4204 unsigned NumElems = SVOp->getValueType(0).getVectorNumElements(); 4205 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, 4206 false /* check zeros from right */, DAG); 4207 unsigned OpSrc; 4208 4209 if (!NumZeros) 4210 return false; 4211 4212 // Considering the elements in the mask that are not consecutive zeros, 4213 // check if they consecutively come from only one of the source vectors. 4214 // 4215 // V1 = {X, A, B, C} 0 4216 // \ \ \ / 4217 // vector_shuffle V1, V2 <1, 2, 3, X> 4218 // 4219 if (!isShuffleMaskConsecutive(SVOp, 4220 0, // Mask Start Index 4221 NumElems-NumZeros-1, // Mask End Index 4222 NumZeros, // Where to start looking in the src vector 4223 NumElems, // Number of elements in vector 4224 OpSrc)) // Which source operand ? 4225 return false; 4226 4227 isLeft = false; 4228 ShAmt = NumZeros; 4229 ShVal = SVOp->getOperand(OpSrc); 4230 return true; 4231} 4232 4233/// isVectorShiftLeft - Returns true if the shuffle can be implemented as a 4234/// logical left shift of a vector. 4235static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4236 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4237 unsigned NumElems = SVOp->getValueType(0).getVectorNumElements(); 4238 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, 4239 true /* check zeros from left */, DAG); 4240 unsigned OpSrc; 4241 4242 if (!NumZeros) 4243 return false; 4244 4245 // Considering the elements in the mask that are not consecutive zeros, 4246 // check if they consecutively come from only one of the source vectors. 4247 // 4248 // 0 { A, B, X, X } = V2 4249 // / \ / / 4250 // vector_shuffle V1, V2 <X, X, 4, 5> 4251 // 4252 if (!isShuffleMaskConsecutive(SVOp, 4253 NumZeros, // Mask Start Index 4254 NumElems-1, // Mask End Index 4255 0, // Where to start looking in the src vector 4256 NumElems, // Number of elements in vector 4257 OpSrc)) // Which source operand ? 4258 return false; 4259 4260 isLeft = true; 4261 ShAmt = NumZeros; 4262 ShVal = SVOp->getOperand(OpSrc); 4263 return true; 4264} 4265 4266/// isVectorShift - Returns true if the shuffle can be implemented as a 4267/// logical left or right shift of a vector. 4268static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4269 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4270 if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) || 4271 isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt)) 4272 return true; 4273 4274 return false; 4275} 4276 4277/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 4278/// 4279static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 4280 unsigned NumNonZero, unsigned NumZero, 4281 SelectionDAG &DAG, 4282 const TargetLowering &TLI) { 4283 if (NumNonZero > 8) 4284 return SDValue(); 4285 4286 DebugLoc dl = Op.getDebugLoc(); 4287 SDValue V(0, 0); 4288 bool First = true; 4289 for (unsigned i = 0; i < 16; ++i) { 4290 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 4291 if (ThisIsNonZero && First) { 4292 if (NumZero) 4293 V = getZeroVector(MVT::v8i16, true, DAG, dl); 4294 else 4295 V = DAG.getUNDEF(MVT::v8i16); 4296 First = false; 4297 } 4298 4299 if ((i & 1) != 0) { 4300 SDValue ThisElt(0, 0), LastElt(0, 0); 4301 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 4302 if (LastIsNonZero) { 4303 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, 4304 MVT::i16, Op.getOperand(i-1)); 4305 } 4306 if (ThisIsNonZero) { 4307 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); 4308 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, 4309 ThisElt, DAG.getConstant(8, MVT::i8)); 4310 if (LastIsNonZero) 4311 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); 4312 } else 4313 ThisElt = LastElt; 4314 4315 if (ThisElt.getNode()) 4316 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, 4317 DAG.getIntPtrConstant(i/2)); 4318 } 4319 } 4320 4321 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); 4322} 4323 4324/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 4325/// 4326static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 4327 unsigned NumNonZero, unsigned NumZero, 4328 SelectionDAG &DAG, 4329 const TargetLowering &TLI) { 4330 if (NumNonZero > 4) 4331 return SDValue(); 4332 4333 DebugLoc dl = Op.getDebugLoc(); 4334 SDValue V(0, 0); 4335 bool First = true; 4336 for (unsigned i = 0; i < 8; ++i) { 4337 bool isNonZero = (NonZeros & (1 << i)) != 0; 4338 if (isNonZero) { 4339 if (First) { 4340 if (NumZero) 4341 V = getZeroVector(MVT::v8i16, true, DAG, dl); 4342 else 4343 V = DAG.getUNDEF(MVT::v8i16); 4344 First = false; 4345 } 4346 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 4347 MVT::v8i16, V, Op.getOperand(i), 4348 DAG.getIntPtrConstant(i)); 4349 } 4350 } 4351 4352 return V; 4353} 4354 4355/// getVShift - Return a vector logical shift node. 4356/// 4357static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, 4358 unsigned NumBits, SelectionDAG &DAG, 4359 const TargetLowering &TLI, DebugLoc dl) { 4360 EVT ShVT = MVT::v2i64; 4361 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 4362 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); 4363 return DAG.getNode(ISD::BITCAST, dl, VT, 4364 DAG.getNode(Opc, dl, ShVT, SrcOp, 4365 DAG.getConstant(NumBits, 4366 TLI.getShiftAmountTy(SrcOp.getValueType())))); 4367} 4368 4369SDValue 4370X86TargetLowering::LowerAsSplatVectorLoad(SDValue SrcOp, EVT VT, DebugLoc dl, 4371 SelectionDAG &DAG) const { 4372 4373 // Check if the scalar load can be widened into a vector load. And if 4374 // the address is "base + cst" see if the cst can be "absorbed" into 4375 // the shuffle mask. 4376 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) { 4377 SDValue Ptr = LD->getBasePtr(); 4378 if (!ISD::isNormalLoad(LD) || LD->isVolatile()) 4379 return SDValue(); 4380 EVT PVT = LD->getValueType(0); 4381 if (PVT != MVT::i32 && PVT != MVT::f32) 4382 return SDValue(); 4383 4384 int FI = -1; 4385 int64_t Offset = 0; 4386 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) { 4387 FI = FINode->getIndex(); 4388 Offset = 0; 4389 } else if (DAG.isBaseWithConstantOffset(Ptr) && 4390 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 4391 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4392 Offset = Ptr.getConstantOperandVal(1); 4393 Ptr = Ptr.getOperand(0); 4394 } else { 4395 return SDValue(); 4396 } 4397 4398 SDValue Chain = LD->getChain(); 4399 // Make sure the stack object alignment is at least 16. 4400 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4401 if (DAG.InferPtrAlignment(Ptr) < 16) { 4402 if (MFI->isFixedObjectIndex(FI)) { 4403 // Can't change the alignment. FIXME: It's possible to compute 4404 // the exact stack offset and reference FI + adjust offset instead. 4405 // If someone *really* cares about this. That's the way to implement it. 4406 return SDValue(); 4407 } else { 4408 MFI->setObjectAlignment(FI, 16); 4409 } 4410 } 4411 4412 // (Offset % 16) must be multiple of 4. Then address is then 4413 // Ptr + (Offset & ~15). 4414 if (Offset < 0) 4415 return SDValue(); 4416 if ((Offset % 16) & 3) 4417 return SDValue(); 4418 int64_t StartOffset = Offset & ~15; 4419 if (StartOffset) 4420 Ptr = DAG.getNode(ISD::ADD, Ptr.getDebugLoc(), Ptr.getValueType(), 4421 Ptr,DAG.getConstant(StartOffset, Ptr.getValueType())); 4422 4423 int EltNo = (Offset - StartOffset) >> 2; 4424 int Mask[4] = { EltNo, EltNo, EltNo, EltNo }; 4425 EVT VT = (PVT == MVT::i32) ? MVT::v4i32 : MVT::v4f32; 4426 SDValue V1 = DAG.getLoad(VT, dl, Chain, Ptr, 4427 LD->getPointerInfo().getWithOffset(StartOffset), 4428 false, false, 0); 4429 // Canonicalize it to a v4i32 shuffle. 4430 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 4431 return DAG.getNode(ISD::BITCAST, dl, VT, 4432 DAG.getVectorShuffle(MVT::v4i32, dl, V1, 4433 DAG.getUNDEF(MVT::v4i32),&Mask[0])); 4434 } 4435 4436 return SDValue(); 4437} 4438 4439/// EltsFromConsecutiveLoads - Given the initializing elements 'Elts' of a 4440/// vector of type 'VT', see if the elements can be replaced by a single large 4441/// load which has the same value as a build_vector whose operands are 'elts'. 4442/// 4443/// Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a 4444/// 4445/// FIXME: we'd also like to handle the case where the last elements are zero 4446/// rather than undef via VZEXT_LOAD, but we do not detect that case today. 4447/// There's even a handy isZeroNode for that purpose. 4448static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts, 4449 DebugLoc &DL, SelectionDAG &DAG) { 4450 EVT EltVT = VT.getVectorElementType(); 4451 unsigned NumElems = Elts.size(); 4452 4453 LoadSDNode *LDBase = NULL; 4454 unsigned LastLoadedElt = -1U; 4455 4456 // For each element in the initializer, see if we've found a load or an undef. 4457 // If we don't find an initial load element, or later load elements are 4458 // non-consecutive, bail out. 4459 for (unsigned i = 0; i < NumElems; ++i) { 4460 SDValue Elt = Elts[i]; 4461 4462 if (!Elt.getNode() || 4463 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) 4464 return SDValue(); 4465 if (!LDBase) { 4466 if (Elt.getNode()->getOpcode() == ISD::UNDEF) 4467 return SDValue(); 4468 LDBase = cast<LoadSDNode>(Elt.getNode()); 4469 LastLoadedElt = i; 4470 continue; 4471 } 4472 if (Elt.getOpcode() == ISD::UNDEF) 4473 continue; 4474 4475 LoadSDNode *LD = cast<LoadSDNode>(Elt); 4476 if (!DAG.isConsecutiveLoad(LD, LDBase, EltVT.getSizeInBits()/8, i)) 4477 return SDValue(); 4478 LastLoadedElt = i; 4479 } 4480 4481 // If we have found an entire vector of loads and undefs, then return a large 4482 // load of the entire vector width starting at the base pointer. If we found 4483 // consecutive loads for the low half, generate a vzext_load node. 4484 if (LastLoadedElt == NumElems - 1) { 4485 if (DAG.InferPtrAlignment(LDBase->getBasePtr()) >= 16) 4486 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), 4487 LDBase->getPointerInfo(), 4488 LDBase->isVolatile(), LDBase->isNonTemporal(), 0); 4489 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), 4490 LDBase->getPointerInfo(), 4491 LDBase->isVolatile(), LDBase->isNonTemporal(), 4492 LDBase->getAlignment()); 4493 } else if (NumElems == 4 && LastLoadedElt == 1) { 4494 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); 4495 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() }; 4496 SDValue ResNode = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, 4497 Ops, 2, MVT::i32, 4498 LDBase->getMemOperand()); 4499 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); 4500 } 4501 return SDValue(); 4502} 4503 4504SDValue 4505X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { 4506 DebugLoc dl = Op.getDebugLoc(); 4507 4508 EVT VT = Op.getValueType(); 4509 EVT ExtVT = VT.getVectorElementType(); 4510 unsigned NumElems = Op.getNumOperands(); 4511 4512 // All zero's: 4513 // - pxor (SSE2), xorps (SSE1), vpxor (128 AVX), xorp[s|d] (256 AVX) 4514 // All one's: 4515 // - pcmpeqd (SSE2 and 128 AVX), fallback to constant pools (256 AVX) 4516 if (ISD::isBuildVectorAllZeros(Op.getNode()) || 4517 ISD::isBuildVectorAllOnes(Op.getNode())) { 4518 // Canonicalize this to <4 x i32> or <8 x 32> (SSE) to 4519 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 4520 // eliminated on x86-32 hosts. 4521 if (Op.getValueType() == MVT::v4i32 || 4522 Op.getValueType() == MVT::v8i32) 4523 return Op; 4524 4525 if (ISD::isBuildVectorAllOnes(Op.getNode())) 4526 return getOnesVector(Op.getValueType(), DAG, dl); 4527 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl); 4528 } 4529 4530 unsigned EVTBits = ExtVT.getSizeInBits(); 4531 4532 unsigned NumZero = 0; 4533 unsigned NumNonZero = 0; 4534 unsigned NonZeros = 0; 4535 bool IsAllConstants = true; 4536 SmallSet<SDValue, 8> Values; 4537 for (unsigned i = 0; i < NumElems; ++i) { 4538 SDValue Elt = Op.getOperand(i); 4539 if (Elt.getOpcode() == ISD::UNDEF) 4540 continue; 4541 Values.insert(Elt); 4542 if (Elt.getOpcode() != ISD::Constant && 4543 Elt.getOpcode() != ISD::ConstantFP) 4544 IsAllConstants = false; 4545 if (X86::isZeroNode(Elt)) 4546 NumZero++; 4547 else { 4548 NonZeros |= (1 << i); 4549 NumNonZero++; 4550 } 4551 } 4552 4553 // All undef vector. Return an UNDEF. All zero vectors were handled above. 4554 if (NumNonZero == 0) 4555 return DAG.getUNDEF(VT); 4556 4557 // Special case for single non-zero, non-undef, element. 4558 if (NumNonZero == 1) { 4559 unsigned Idx = CountTrailingZeros_32(NonZeros); 4560 SDValue Item = Op.getOperand(Idx); 4561 4562 // If this is an insertion of an i64 value on x86-32, and if the top bits of 4563 // the value are obviously zero, truncate the value to i32 and do the 4564 // insertion that way. Only do this if the value is non-constant or if the 4565 // value is a constant being inserted into element 0. It is cheaper to do 4566 // a constant pool load than it is to do a movd + shuffle. 4567 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && 4568 (!IsAllConstants || Idx == 0)) { 4569 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 4570 // Handle SSE only. 4571 assert(VT == MVT::v2i64 && "Expected an SSE value type!"); 4572 EVT VecVT = MVT::v4i32; 4573 unsigned VecElts = 4; 4574 4575 // Truncate the value (which may itself be a constant) to i32, and 4576 // convert it to a vector with movd (S2V+shuffle to zero extend). 4577 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); 4578 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); 4579 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 4580 Subtarget->hasSSE2(), DAG); 4581 4582 // Now we have our 32-bit value zero extended in the low element of 4583 // a vector. If Idx != 0, swizzle it into place. 4584 if (Idx != 0) { 4585 SmallVector<int, 4> Mask; 4586 Mask.push_back(Idx); 4587 for (unsigned i = 1; i != VecElts; ++i) 4588 Mask.push_back(i); 4589 Item = DAG.getVectorShuffle(VecVT, dl, Item, 4590 DAG.getUNDEF(Item.getValueType()), 4591 &Mask[0]); 4592 } 4593 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Item); 4594 } 4595 } 4596 4597 // If we have a constant or non-constant insertion into the low element of 4598 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 4599 // the rest of the elements. This will be matched as movd/movq/movss/movsd 4600 // depending on what the source datatype is. 4601 if (Idx == 0) { 4602 if (NumZero == 0) { 4603 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 4604 } else if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || 4605 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { 4606 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 4607 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 4608 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget->hasSSE2(), 4609 DAG); 4610 } else if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { 4611 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); 4612 assert(VT.getSizeInBits() == 128 && "Expected an SSE value type!"); 4613 EVT MiddleVT = MVT::v4i32; 4614 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item); 4615 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 4616 Subtarget->hasSSE2(), DAG); 4617 return DAG.getNode(ISD::BITCAST, dl, VT, Item); 4618 } 4619 } 4620 4621 // Is it a vector logical left shift? 4622 if (NumElems == 2 && Idx == 1 && 4623 X86::isZeroNode(Op.getOperand(0)) && 4624 !X86::isZeroNode(Op.getOperand(1))) { 4625 unsigned NumBits = VT.getSizeInBits(); 4626 return getVShift(true, VT, 4627 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 4628 VT, Op.getOperand(1)), 4629 NumBits/2, DAG, *this, dl); 4630 } 4631 4632 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 4633 return SDValue(); 4634 4635 // Otherwise, if this is a vector with i32 or f32 elements, and the element 4636 // is a non-constant being inserted into an element other than the low one, 4637 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 4638 // movd/movss) to move this into the low element, then shuffle it into 4639 // place. 4640 if (EVTBits == 32) { 4641 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 4642 4643 // Turn it into a shuffle of zero and zero-extended scalar to vector. 4644 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 4645 Subtarget->hasSSE2(), DAG); 4646 SmallVector<int, 8> MaskVec; 4647 for (unsigned i = 0; i < NumElems; i++) 4648 MaskVec.push_back(i == Idx ? 0 : 1); 4649 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]); 4650 } 4651 } 4652 4653 // Splat is obviously ok. Let legalizer expand it to a shuffle. 4654 if (Values.size() == 1) { 4655 if (EVTBits == 32) { 4656 // Instead of a shuffle like this: 4657 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> 4658 // Check if it's possible to issue this instead. 4659 // shuffle (vload ptr)), undef, <1, 1, 1, 1> 4660 unsigned Idx = CountTrailingZeros_32(NonZeros); 4661 SDValue Item = Op.getOperand(Idx); 4662 if (Op.getNode()->isOnlyUserOf(Item.getNode())) 4663 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); 4664 } 4665 return SDValue(); 4666 } 4667 4668 // A vector full of immediates; various special cases are already 4669 // handled, so this is best done with a single constant-pool load. 4670 if (IsAllConstants) 4671 return SDValue(); 4672 4673 // For AVX-length vectors, build the individual 128-bit pieces and use 4674 // shuffles to put them in place. 4675 if (VT.getSizeInBits() == 256 && !ISD::isBuildVectorAllZeros(Op.getNode())) { 4676 SmallVector<SDValue, 32> V; 4677 for (unsigned i = 0; i < NumElems; ++i) 4678 V.push_back(Op.getOperand(i)); 4679 4680 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2); 4681 4682 // Build both the lower and upper subvector. 4683 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[0], NumElems/2); 4684 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[NumElems / 2], 4685 NumElems/2); 4686 4687 // Recreate the wider vector with the lower and upper part. 4688 SDValue Vec = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT), Upper, 4689 DAG.getConstant(NumElems/2, MVT::i32), DAG, dl); 4690 return Insert128BitVector(Vec, Lower, DAG.getConstant(0, MVT::i32), 4691 DAG, dl); 4692 } 4693 4694 // Let legalizer expand 2-wide build_vectors. 4695 if (EVTBits == 64) { 4696 if (NumNonZero == 1) { 4697 // One half is zero or undef. 4698 unsigned Idx = CountTrailingZeros_32(NonZeros); 4699 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, 4700 Op.getOperand(Idx)); 4701 return getShuffleVectorZeroOrUndef(V2, Idx, true, 4702 Subtarget->hasSSE2(), DAG); 4703 } 4704 return SDValue(); 4705 } 4706 4707 // If element VT is < 32 bits, convert it to inserts into a zero vector. 4708 if (EVTBits == 8 && NumElems == 16) { 4709 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 4710 *this); 4711 if (V.getNode()) return V; 4712 } 4713 4714 if (EVTBits == 16 && NumElems == 8) { 4715 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 4716 *this); 4717 if (V.getNode()) return V; 4718 } 4719 4720 // If element VT is == 32 bits, turn it into a number of shuffles. 4721 SmallVector<SDValue, 8> V; 4722 V.resize(NumElems); 4723 if (NumElems == 4 && NumZero > 0) { 4724 for (unsigned i = 0; i < 4; ++i) { 4725 bool isZero = !(NonZeros & (1 << i)); 4726 if (isZero) 4727 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); 4728 else 4729 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 4730 } 4731 4732 for (unsigned i = 0; i < 2; ++i) { 4733 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 4734 default: break; 4735 case 0: 4736 V[i] = V[i*2]; // Must be a zero vector. 4737 break; 4738 case 1: 4739 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); 4740 break; 4741 case 2: 4742 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); 4743 break; 4744 case 3: 4745 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); 4746 break; 4747 } 4748 } 4749 4750 SmallVector<int, 8> MaskVec; 4751 bool Reverse = (NonZeros & 0x3) == 2; 4752 for (unsigned i = 0; i < 2; ++i) 4753 MaskVec.push_back(Reverse ? 1-i : i); 4754 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 4755 for (unsigned i = 0; i < 2; ++i) 4756 MaskVec.push_back(Reverse ? 1-i+NumElems : i+NumElems); 4757 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); 4758 } 4759 4760 if (Values.size() > 1 && VT.getSizeInBits() == 128) { 4761 // Check for a build vector of consecutive loads. 4762 for (unsigned i = 0; i < NumElems; ++i) 4763 V[i] = Op.getOperand(i); 4764 4765 // Check for elements which are consecutive loads. 4766 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG); 4767 if (LD.getNode()) 4768 return LD; 4769 4770 // For SSE 4.1, use insertps to put the high elements into the low element. 4771 if (getSubtarget()->hasSSE41()) { 4772 SDValue Result; 4773 if (Op.getOperand(0).getOpcode() != ISD::UNDEF) 4774 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); 4775 else 4776 Result = DAG.getUNDEF(VT); 4777 4778 for (unsigned i = 1; i < NumElems; ++i) { 4779 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue; 4780 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, 4781 Op.getOperand(i), DAG.getIntPtrConstant(i)); 4782 } 4783 return Result; 4784 } 4785 4786 // Otherwise, expand into a number of unpckl*, start by extending each of 4787 // our (non-undef) elements to the full vector width with the element in the 4788 // bottom slot of the vector (which generates no code for SSE). 4789 for (unsigned i = 0; i < NumElems; ++i) { 4790 if (Op.getOperand(i).getOpcode() != ISD::UNDEF) 4791 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 4792 else 4793 V[i] = DAG.getUNDEF(VT); 4794 } 4795 4796 // Next, we iteratively mix elements, e.g. for v4f32: 4797 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 4798 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 4799 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 4800 unsigned EltStride = NumElems >> 1; 4801 while (EltStride != 0) { 4802 for (unsigned i = 0; i < EltStride; ++i) { 4803 // If V[i+EltStride] is undef and this is the first round of mixing, 4804 // then it is safe to just drop this shuffle: V[i] is already in the 4805 // right place, the one element (since it's the first round) being 4806 // inserted as undef can be dropped. This isn't safe for successive 4807 // rounds because they will permute elements within both vectors. 4808 if (V[i+EltStride].getOpcode() == ISD::UNDEF && 4809 EltStride == NumElems/2) 4810 continue; 4811 4812 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]); 4813 } 4814 EltStride >>= 1; 4815 } 4816 return V[0]; 4817 } 4818 return SDValue(); 4819} 4820 4821SDValue 4822X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const { 4823 // We support concatenate two MMX registers and place them in a MMX 4824 // register. This is better than doing a stack convert. 4825 DebugLoc dl = Op.getDebugLoc(); 4826 EVT ResVT = Op.getValueType(); 4827 assert(Op.getNumOperands() == 2); 4828 assert(ResVT == MVT::v2i64 || ResVT == MVT::v4i32 || 4829 ResVT == MVT::v8i16 || ResVT == MVT::v16i8); 4830 int Mask[2]; 4831 SDValue InVec = DAG.getNode(ISD::BITCAST,dl, MVT::v1i64, Op.getOperand(0)); 4832 SDValue VecOp = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); 4833 InVec = Op.getOperand(1); 4834 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) { 4835 unsigned NumElts = ResVT.getVectorNumElements(); 4836 VecOp = DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); 4837 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ResVT, VecOp, 4838 InVec.getOperand(0), DAG.getIntPtrConstant(NumElts/2+1)); 4839 } else { 4840 InVec = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, InVec); 4841 SDValue VecOp2 = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); 4842 Mask[0] = 0; Mask[1] = 2; 4843 VecOp = DAG.getVectorShuffle(MVT::v2i64, dl, VecOp, VecOp2, Mask); 4844 } 4845 return DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); 4846} 4847 4848// v8i16 shuffles - Prefer shuffles in the following order: 4849// 1. [all] pshuflw, pshufhw, optional move 4850// 2. [ssse3] 1 x pshufb 4851// 3. [ssse3] 2 x pshufb + 1 x por 4852// 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw) 4853SDValue 4854X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op, 4855 SelectionDAG &DAG) const { 4856 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 4857 SDValue V1 = SVOp->getOperand(0); 4858 SDValue V2 = SVOp->getOperand(1); 4859 DebugLoc dl = SVOp->getDebugLoc(); 4860 SmallVector<int, 8> MaskVals; 4861 4862 // Determine if more than 1 of the words in each of the low and high quadwords 4863 // of the result come from the same quadword of one of the two inputs. Undef 4864 // mask values count as coming from any quadword, for better codegen. 4865 SmallVector<unsigned, 4> LoQuad(4); 4866 SmallVector<unsigned, 4> HiQuad(4); 4867 BitVector InputQuads(4); 4868 for (unsigned i = 0; i < 8; ++i) { 4869 SmallVectorImpl<unsigned> &Quad = i < 4 ? LoQuad : HiQuad; 4870 int EltIdx = SVOp->getMaskElt(i); 4871 MaskVals.push_back(EltIdx); 4872 if (EltIdx < 0) { 4873 ++Quad[0]; 4874 ++Quad[1]; 4875 ++Quad[2]; 4876 ++Quad[3]; 4877 continue; 4878 } 4879 ++Quad[EltIdx / 4]; 4880 InputQuads.set(EltIdx / 4); 4881 } 4882 4883 int BestLoQuad = -1; 4884 unsigned MaxQuad = 1; 4885 for (unsigned i = 0; i < 4; ++i) { 4886 if (LoQuad[i] > MaxQuad) { 4887 BestLoQuad = i; 4888 MaxQuad = LoQuad[i]; 4889 } 4890 } 4891 4892 int BestHiQuad = -1; 4893 MaxQuad = 1; 4894 for (unsigned i = 0; i < 4; ++i) { 4895 if (HiQuad[i] > MaxQuad) { 4896 BestHiQuad = i; 4897 MaxQuad = HiQuad[i]; 4898 } 4899 } 4900 4901 // For SSSE3, If all 8 words of the result come from only 1 quadword of each 4902 // of the two input vectors, shuffle them into one input vector so only a 4903 // single pshufb instruction is necessary. If There are more than 2 input 4904 // quads, disable the next transformation since it does not help SSSE3. 4905 bool V1Used = InputQuads[0] || InputQuads[1]; 4906 bool V2Used = InputQuads[2] || InputQuads[3]; 4907 if (Subtarget->hasSSSE3()) { 4908 if (InputQuads.count() == 2 && V1Used && V2Used) { 4909 BestLoQuad = InputQuads.find_first(); 4910 BestHiQuad = InputQuads.find_next(BestLoQuad); 4911 } 4912 if (InputQuads.count() > 2) { 4913 BestLoQuad = -1; 4914 BestHiQuad = -1; 4915 } 4916 } 4917 4918 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update 4919 // the shuffle mask. If a quad is scored as -1, that means that it contains 4920 // words from all 4 input quadwords. 4921 SDValue NewV; 4922 if (BestLoQuad >= 0 || BestHiQuad >= 0) { 4923 SmallVector<int, 8> MaskV; 4924 MaskV.push_back(BestLoQuad < 0 ? 0 : BestLoQuad); 4925 MaskV.push_back(BestHiQuad < 0 ? 1 : BestHiQuad); 4926 NewV = DAG.getVectorShuffle(MVT::v2i64, dl, 4927 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1), 4928 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]); 4929 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV); 4930 4931 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the 4932 // source words for the shuffle, to aid later transformations. 4933 bool AllWordsInNewV = true; 4934 bool InOrder[2] = { true, true }; 4935 for (unsigned i = 0; i != 8; ++i) { 4936 int idx = MaskVals[i]; 4937 if (idx != (int)i) 4938 InOrder[i/4] = false; 4939 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad) 4940 continue; 4941 AllWordsInNewV = false; 4942 break; 4943 } 4944 4945 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV; 4946 if (AllWordsInNewV) { 4947 for (int i = 0; i != 8; ++i) { 4948 int idx = MaskVals[i]; 4949 if (idx < 0) 4950 continue; 4951 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4; 4952 if ((idx != i) && idx < 4) 4953 pshufhw = false; 4954 if ((idx != i) && idx > 3) 4955 pshuflw = false; 4956 } 4957 V1 = NewV; 4958 V2Used = false; 4959 BestLoQuad = 0; 4960 BestHiQuad = 1; 4961 } 4962 4963 // If we've eliminated the use of V2, and the new mask is a pshuflw or 4964 // pshufhw, that's as cheap as it gets. Return the new shuffle. 4965 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) { 4966 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW; 4967 unsigned TargetMask = 0; 4968 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, 4969 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); 4970 TargetMask = pshufhw ? X86::getShufflePSHUFHWImmediate(NewV.getNode()): 4971 X86::getShufflePSHUFLWImmediate(NewV.getNode()); 4972 V1 = NewV.getOperand(0); 4973 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG); 4974 } 4975 } 4976 4977 // If we have SSSE3, and all words of the result are from 1 input vector, 4978 // case 2 is generated, otherwise case 3 is generated. If no SSSE3 4979 // is present, fall back to case 4. 4980 if (Subtarget->hasSSSE3()) { 4981 SmallVector<SDValue,16> pshufbMask; 4982 4983 // If we have elements from both input vectors, set the high bit of the 4984 // shuffle mask element to zero out elements that come from V2 in the V1 4985 // mask, and elements that come from V1 in the V2 mask, so that the two 4986 // results can be OR'd together. 4987 bool TwoInputs = V1Used && V2Used; 4988 for (unsigned i = 0; i != 8; ++i) { 4989 int EltIdx = MaskVals[i] * 2; 4990 if (TwoInputs && (EltIdx >= 16)) { 4991 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 4992 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 4993 continue; 4994 } 4995 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 4996 pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8)); 4997 } 4998 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V1); 4999 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 5000 DAG.getNode(ISD::BUILD_VECTOR, dl, 5001 MVT::v16i8, &pshufbMask[0], 16)); 5002 if (!TwoInputs) 5003 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5004 5005 // Calculate the shuffle mask for the second input, shuffle it, and 5006 // OR it with the first shuffled input. 5007 pshufbMask.clear(); 5008 for (unsigned i = 0; i != 8; ++i) { 5009 int EltIdx = MaskVals[i] * 2; 5010 if (EltIdx < 16) { 5011 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5012 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5013 continue; 5014 } 5015 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 5016 pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8)); 5017 } 5018 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V2); 5019 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 5020 DAG.getNode(ISD::BUILD_VECTOR, dl, 5021 MVT::v16i8, &pshufbMask[0], 16)); 5022 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 5023 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5024 } 5025 5026 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order, 5027 // and update MaskVals with new element order. 5028 BitVector InOrder(8); 5029 if (BestLoQuad >= 0) { 5030 SmallVector<int, 8> MaskV; 5031 for (int i = 0; i != 4; ++i) { 5032 int idx = MaskVals[i]; 5033 if (idx < 0) { 5034 MaskV.push_back(-1); 5035 InOrder.set(i); 5036 } else if ((idx / 4) == BestLoQuad) { 5037 MaskV.push_back(idx & 3); 5038 InOrder.set(i); 5039 } else { 5040 MaskV.push_back(-1); 5041 } 5042 } 5043 for (unsigned i = 4; i != 8; ++i) 5044 MaskV.push_back(i); 5045 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 5046 &MaskV[0]); 5047 5048 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3()) 5049 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16, 5050 NewV.getOperand(0), 5051 X86::getShufflePSHUFLWImmediate(NewV.getNode()), 5052 DAG); 5053 } 5054 5055 // If BestHi >= 0, generate a pshufhw to put the high elements in order, 5056 // and update MaskVals with the new element order. 5057 if (BestHiQuad >= 0) { 5058 SmallVector<int, 8> MaskV; 5059 for (unsigned i = 0; i != 4; ++i) 5060 MaskV.push_back(i); 5061 for (unsigned i = 4; i != 8; ++i) { 5062 int idx = MaskVals[i]; 5063 if (idx < 0) { 5064 MaskV.push_back(-1); 5065 InOrder.set(i); 5066 } else if ((idx / 4) == BestHiQuad) { 5067 MaskV.push_back((idx & 3) + 4); 5068 InOrder.set(i); 5069 } else { 5070 MaskV.push_back(-1); 5071 } 5072 } 5073 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 5074 &MaskV[0]); 5075 5076 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3()) 5077 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16, 5078 NewV.getOperand(0), 5079 X86::getShufflePSHUFHWImmediate(NewV.getNode()), 5080 DAG); 5081 } 5082 5083 // In case BestHi & BestLo were both -1, which means each quadword has a word 5084 // from each of the four input quadwords, calculate the InOrder bitvector now 5085 // before falling through to the insert/extract cleanup. 5086 if (BestLoQuad == -1 && BestHiQuad == -1) { 5087 NewV = V1; 5088 for (int i = 0; i != 8; ++i) 5089 if (MaskVals[i] < 0 || MaskVals[i] == i) 5090 InOrder.set(i); 5091 } 5092 5093 // The other elements are put in the right place using pextrw and pinsrw. 5094 for (unsigned i = 0; i != 8; ++i) { 5095 if (InOrder[i]) 5096 continue; 5097 int EltIdx = MaskVals[i]; 5098 if (EltIdx < 0) 5099 continue; 5100 SDValue ExtOp = (EltIdx < 8) 5101 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, 5102 DAG.getIntPtrConstant(EltIdx)) 5103 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, 5104 DAG.getIntPtrConstant(EltIdx - 8)); 5105 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, 5106 DAG.getIntPtrConstant(i)); 5107 } 5108 return NewV; 5109} 5110 5111// v16i8 shuffles - Prefer shuffles in the following order: 5112// 1. [ssse3] 1 x pshufb 5113// 2. [ssse3] 2 x pshufb + 1 x por 5114// 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw 5115static 5116SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp, 5117 SelectionDAG &DAG, 5118 const X86TargetLowering &TLI) { 5119 SDValue V1 = SVOp->getOperand(0); 5120 SDValue V2 = SVOp->getOperand(1); 5121 DebugLoc dl = SVOp->getDebugLoc(); 5122 SmallVector<int, 16> MaskVals; 5123 SVOp->getMask(MaskVals); 5124 5125 // If we have SSSE3, case 1 is generated when all result bytes come from 5126 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is 5127 // present, fall back to case 3. 5128 // FIXME: kill V2Only once shuffles are canonizalized by getNode. 5129 bool V1Only = true; 5130 bool V2Only = true; 5131 for (unsigned i = 0; i < 16; ++i) { 5132 int EltIdx = MaskVals[i]; 5133 if (EltIdx < 0) 5134 continue; 5135 if (EltIdx < 16) 5136 V2Only = false; 5137 else 5138 V1Only = false; 5139 } 5140 5141 // If SSSE3, use 1 pshufb instruction per vector with elements in the result. 5142 if (TLI.getSubtarget()->hasSSSE3()) { 5143 SmallVector<SDValue,16> pshufbMask; 5144 5145 // If all result elements are from one input vector, then only translate 5146 // undef mask values to 0x80 (zero out result) in the pshufb mask. 5147 // 5148 // Otherwise, we have elements from both input vectors, and must zero out 5149 // elements that come from V2 in the first mask, and V1 in the second mask 5150 // so that we can OR them together. 5151 bool TwoInputs = !(V1Only || V2Only); 5152 for (unsigned i = 0; i != 16; ++i) { 5153 int EltIdx = MaskVals[i]; 5154 if (EltIdx < 0 || (TwoInputs && EltIdx >= 16)) { 5155 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5156 continue; 5157 } 5158 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 5159 } 5160 // If all the elements are from V2, assign it to V1 and return after 5161 // building the first pshufb. 5162 if (V2Only) 5163 V1 = V2; 5164 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 5165 DAG.getNode(ISD::BUILD_VECTOR, dl, 5166 MVT::v16i8, &pshufbMask[0], 16)); 5167 if (!TwoInputs) 5168 return V1; 5169 5170 // Calculate the shuffle mask for the second input, shuffle it, and 5171 // OR it with the first shuffled input. 5172 pshufbMask.clear(); 5173 for (unsigned i = 0; i != 16; ++i) { 5174 int EltIdx = MaskVals[i]; 5175 if (EltIdx < 16) { 5176 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5177 continue; 5178 } 5179 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 5180 } 5181 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 5182 DAG.getNode(ISD::BUILD_VECTOR, dl, 5183 MVT::v16i8, &pshufbMask[0], 16)); 5184 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 5185 } 5186 5187 // No SSSE3 - Calculate in place words and then fix all out of place words 5188 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from 5189 // the 16 different words that comprise the two doublequadword input vectors. 5190 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5191 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); 5192 SDValue NewV = V2Only ? V2 : V1; 5193 for (int i = 0; i != 8; ++i) { 5194 int Elt0 = MaskVals[i*2]; 5195 int Elt1 = MaskVals[i*2+1]; 5196 5197 // This word of the result is all undef, skip it. 5198 if (Elt0 < 0 && Elt1 < 0) 5199 continue; 5200 5201 // This word of the result is already in the correct place, skip it. 5202 if (V1Only && (Elt0 == i*2) && (Elt1 == i*2+1)) 5203 continue; 5204 if (V2Only && (Elt0 == i*2+16) && (Elt1 == i*2+17)) 5205 continue; 5206 5207 SDValue Elt0Src = Elt0 < 16 ? V1 : V2; 5208 SDValue Elt1Src = Elt1 < 16 ? V1 : V2; 5209 SDValue InsElt; 5210 5211 // If Elt0 and Elt1 are defined, are consecutive, and can be load 5212 // using a single extract together, load it and store it. 5213 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) { 5214 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 5215 DAG.getIntPtrConstant(Elt1 / 2)); 5216 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 5217 DAG.getIntPtrConstant(i)); 5218 continue; 5219 } 5220 5221 // If Elt1 is defined, extract it from the appropriate source. If the 5222 // source byte is not also odd, shift the extracted word left 8 bits 5223 // otherwise clear the bottom 8 bits if we need to do an or. 5224 if (Elt1 >= 0) { 5225 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 5226 DAG.getIntPtrConstant(Elt1 / 2)); 5227 if ((Elt1 & 1) == 0) 5228 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, 5229 DAG.getConstant(8, 5230 TLI.getShiftAmountTy(InsElt.getValueType()))); 5231 else if (Elt0 >= 0) 5232 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, 5233 DAG.getConstant(0xFF00, MVT::i16)); 5234 } 5235 // If Elt0 is defined, extract it from the appropriate source. If the 5236 // source byte is not also even, shift the extracted word right 8 bits. If 5237 // Elt1 was also defined, OR the extracted values together before 5238 // inserting them in the result. 5239 if (Elt0 >= 0) { 5240 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, 5241 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2)); 5242 if ((Elt0 & 1) != 0) 5243 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, 5244 DAG.getConstant(8, 5245 TLI.getShiftAmountTy(InsElt0.getValueType()))); 5246 else if (Elt1 >= 0) 5247 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, 5248 DAG.getConstant(0x00FF, MVT::i16)); 5249 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) 5250 : InsElt0; 5251 } 5252 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 5253 DAG.getIntPtrConstant(i)); 5254 } 5255 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV); 5256} 5257 5258/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 5259/// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be 5260/// done when every pair / quad of shuffle mask elements point to elements in 5261/// the right sequence. e.g. 5262/// vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15> 5263static 5264SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp, 5265 SelectionDAG &DAG, DebugLoc dl) { 5266 EVT VT = SVOp->getValueType(0); 5267 SDValue V1 = SVOp->getOperand(0); 5268 SDValue V2 = SVOp->getOperand(1); 5269 unsigned NumElems = VT.getVectorNumElements(); 5270 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 5271 EVT NewVT; 5272 switch (VT.getSimpleVT().SimpleTy) { 5273 default: assert(false && "Unexpected!"); 5274 case MVT::v4f32: NewVT = MVT::v2f64; break; 5275 case MVT::v4i32: NewVT = MVT::v2i64; break; 5276 case MVT::v8i16: NewVT = MVT::v4i32; break; 5277 case MVT::v16i8: NewVT = MVT::v4i32; break; 5278 } 5279 5280 int Scale = NumElems / NewWidth; 5281 SmallVector<int, 8> MaskVec; 5282 for (unsigned i = 0; i < NumElems; i += Scale) { 5283 int StartIdx = -1; 5284 for (int j = 0; j < Scale; ++j) { 5285 int EltIdx = SVOp->getMaskElt(i+j); 5286 if (EltIdx < 0) 5287 continue; 5288 if (StartIdx == -1) 5289 StartIdx = EltIdx - (EltIdx % Scale); 5290 if (EltIdx != StartIdx + j) 5291 return SDValue(); 5292 } 5293 if (StartIdx == -1) 5294 MaskVec.push_back(-1); 5295 else 5296 MaskVec.push_back(StartIdx / Scale); 5297 } 5298 5299 V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1); 5300 V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2); 5301 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]); 5302} 5303 5304/// getVZextMovL - Return a zero-extending vector move low node. 5305/// 5306static SDValue getVZextMovL(EVT VT, EVT OpVT, 5307 SDValue SrcOp, SelectionDAG &DAG, 5308 const X86Subtarget *Subtarget, DebugLoc dl) { 5309 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 5310 LoadSDNode *LD = NULL; 5311 if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) 5312 LD = dyn_cast<LoadSDNode>(SrcOp); 5313 if (!LD) { 5314 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 5315 // instead. 5316 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 5317 if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) && 5318 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 5319 SrcOp.getOperand(0).getOpcode() == ISD::BITCAST && 5320 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) { 5321 // PR2108 5322 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 5323 return DAG.getNode(ISD::BITCAST, dl, VT, 5324 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 5325 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 5326 OpVT, 5327 SrcOp.getOperand(0) 5328 .getOperand(0)))); 5329 } 5330 } 5331 } 5332 5333 return DAG.getNode(ISD::BITCAST, dl, VT, 5334 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 5335 DAG.getNode(ISD::BITCAST, dl, 5336 OpVT, SrcOp))); 5337} 5338 5339/// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles 5340/// which could not be matched by any known target speficic shuffle 5341static SDValue 5342LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 5343 return SDValue(); 5344} 5345 5346/// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with 5347/// 4 elements, and match them with several different shuffle types. 5348static SDValue 5349LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 5350 SDValue V1 = SVOp->getOperand(0); 5351 SDValue V2 = SVOp->getOperand(1); 5352 DebugLoc dl = SVOp->getDebugLoc(); 5353 EVT VT = SVOp->getValueType(0); 5354 5355 assert(VT.getSizeInBits() == 128 && "Unsupported vector size"); 5356 5357 SmallVector<std::pair<int, int>, 8> Locs; 5358 Locs.resize(4); 5359 SmallVector<int, 8> Mask1(4U, -1); 5360 SmallVector<int, 8> PermMask; 5361 SVOp->getMask(PermMask); 5362 5363 unsigned NumHi = 0; 5364 unsigned NumLo = 0; 5365 for (unsigned i = 0; i != 4; ++i) { 5366 int Idx = PermMask[i]; 5367 if (Idx < 0) { 5368 Locs[i] = std::make_pair(-1, -1); 5369 } else { 5370 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!"); 5371 if (Idx < 4) { 5372 Locs[i] = std::make_pair(0, NumLo); 5373 Mask1[NumLo] = Idx; 5374 NumLo++; 5375 } else { 5376 Locs[i] = std::make_pair(1, NumHi); 5377 if (2+NumHi < 4) 5378 Mask1[2+NumHi] = Idx; 5379 NumHi++; 5380 } 5381 } 5382 } 5383 5384 if (NumLo <= 2 && NumHi <= 2) { 5385 // If no more than two elements come from either vector. This can be 5386 // implemented with two shuffles. First shuffle gather the elements. 5387 // The second shuffle, which takes the first shuffle as both of its 5388 // vector operands, put the elements into the right order. 5389 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 5390 5391 SmallVector<int, 8> Mask2(4U, -1); 5392 5393 for (unsigned i = 0; i != 4; ++i) { 5394 if (Locs[i].first == -1) 5395 continue; 5396 else { 5397 unsigned Idx = (i < 2) ? 0 : 4; 5398 Idx += Locs[i].first * 2 + Locs[i].second; 5399 Mask2[i] = Idx; 5400 } 5401 } 5402 5403 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); 5404 } else if (NumLo == 3 || NumHi == 3) { 5405 // Otherwise, we must have three elements from one vector, call it X, and 5406 // one element from the other, call it Y. First, use a shufps to build an 5407 // intermediate vector with the one element from Y and the element from X 5408 // that will be in the same half in the final destination (the indexes don't 5409 // matter). Then, use a shufps to build the final vector, taking the half 5410 // containing the element from Y from the intermediate, and the other half 5411 // from X. 5412 if (NumHi == 3) { 5413 // Normalize it so the 3 elements come from V1. 5414 CommuteVectorShuffleMask(PermMask, VT); 5415 std::swap(V1, V2); 5416 } 5417 5418 // Find the element from V2. 5419 unsigned HiIndex; 5420 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 5421 int Val = PermMask[HiIndex]; 5422 if (Val < 0) 5423 continue; 5424 if (Val >= 4) 5425 break; 5426 } 5427 5428 Mask1[0] = PermMask[HiIndex]; 5429 Mask1[1] = -1; 5430 Mask1[2] = PermMask[HiIndex^1]; 5431 Mask1[3] = -1; 5432 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 5433 5434 if (HiIndex >= 2) { 5435 Mask1[0] = PermMask[0]; 5436 Mask1[1] = PermMask[1]; 5437 Mask1[2] = HiIndex & 1 ? 6 : 4; 5438 Mask1[3] = HiIndex & 1 ? 4 : 6; 5439 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 5440 } else { 5441 Mask1[0] = HiIndex & 1 ? 2 : 0; 5442 Mask1[1] = HiIndex & 1 ? 0 : 2; 5443 Mask1[2] = PermMask[2]; 5444 Mask1[3] = PermMask[3]; 5445 if (Mask1[2] >= 0) 5446 Mask1[2] += 4; 5447 if (Mask1[3] >= 0) 5448 Mask1[3] += 4; 5449 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); 5450 } 5451 } 5452 5453 // Break it into (shuffle shuffle_hi, shuffle_lo). 5454 Locs.clear(); 5455 Locs.resize(4); 5456 SmallVector<int,8> LoMask(4U, -1); 5457 SmallVector<int,8> HiMask(4U, -1); 5458 5459 SmallVector<int,8> *MaskPtr = &LoMask; 5460 unsigned MaskIdx = 0; 5461 unsigned LoIdx = 0; 5462 unsigned HiIdx = 2; 5463 for (unsigned i = 0; i != 4; ++i) { 5464 if (i == 2) { 5465 MaskPtr = &HiMask; 5466 MaskIdx = 1; 5467 LoIdx = 0; 5468 HiIdx = 2; 5469 } 5470 int Idx = PermMask[i]; 5471 if (Idx < 0) { 5472 Locs[i] = std::make_pair(-1, -1); 5473 } else if (Idx < 4) { 5474 Locs[i] = std::make_pair(MaskIdx, LoIdx); 5475 (*MaskPtr)[LoIdx] = Idx; 5476 LoIdx++; 5477 } else { 5478 Locs[i] = std::make_pair(MaskIdx, HiIdx); 5479 (*MaskPtr)[HiIdx] = Idx; 5480 HiIdx++; 5481 } 5482 } 5483 5484 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); 5485 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); 5486 SmallVector<int, 8> MaskOps; 5487 for (unsigned i = 0; i != 4; ++i) { 5488 if (Locs[i].first == -1) { 5489 MaskOps.push_back(-1); 5490 } else { 5491 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 5492 MaskOps.push_back(Idx); 5493 } 5494 } 5495 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); 5496} 5497 5498static bool MayFoldVectorLoad(SDValue V) { 5499 if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) 5500 V = V.getOperand(0); 5501 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) 5502 V = V.getOperand(0); 5503 if (MayFoldLoad(V)) 5504 return true; 5505 return false; 5506} 5507 5508// FIXME: the version above should always be used. Since there's 5509// a bug where several vector shuffles can't be folded because the 5510// DAG is not updated during lowering and a node claims to have two 5511// uses while it only has one, use this version, and let isel match 5512// another instruction if the load really happens to have more than 5513// one use. Remove this version after this bug get fixed. 5514// rdar://8434668, PR8156 5515static bool RelaxedMayFoldVectorLoad(SDValue V) { 5516 if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) 5517 V = V.getOperand(0); 5518 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) 5519 V = V.getOperand(0); 5520 if (ISD::isNormalLoad(V.getNode())) 5521 return true; 5522 return false; 5523} 5524 5525/// CanFoldShuffleIntoVExtract - Check if the current shuffle is used by 5526/// a vector extract, and if both can be later optimized into a single load. 5527/// This is done in visitEXTRACT_VECTOR_ELT and the conditions are checked 5528/// here because otherwise a target specific shuffle node is going to be 5529/// emitted for this shuffle, and the optimization not done. 5530/// FIXME: This is probably not the best approach, but fix the problem 5531/// until the right path is decided. 5532static 5533bool CanXFormVExtractWithShuffleIntoLoad(SDValue V, SelectionDAG &DAG, 5534 const TargetLowering &TLI) { 5535 EVT VT = V.getValueType(); 5536 ShuffleVectorSDNode *SVOp = dyn_cast<ShuffleVectorSDNode>(V); 5537 5538 // Be sure that the vector shuffle is present in a pattern like this: 5539 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), c) -> (f32 load $addr) 5540 if (!V.hasOneUse()) 5541 return false; 5542 5543 SDNode *N = *V.getNode()->use_begin(); 5544 if (N->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 5545 return false; 5546 5547 SDValue EltNo = N->getOperand(1); 5548 if (!isa<ConstantSDNode>(EltNo)) 5549 return false; 5550 5551 // If the bit convert changed the number of elements, it is unsafe 5552 // to examine the mask. 5553 bool HasShuffleIntoBitcast = false; 5554 if (V.getOpcode() == ISD::BITCAST) { 5555 EVT SrcVT = V.getOperand(0).getValueType(); 5556 if (SrcVT.getVectorNumElements() != VT.getVectorNumElements()) 5557 return false; 5558 V = V.getOperand(0); 5559 HasShuffleIntoBitcast = true; 5560 } 5561 5562 // Select the input vector, guarding against out of range extract vector. 5563 unsigned NumElems = VT.getVectorNumElements(); 5564 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 5565 int Idx = (Elt > NumElems) ? -1 : SVOp->getMaskElt(Elt); 5566 V = (Idx < (int)NumElems) ? V.getOperand(0) : V.getOperand(1); 5567 5568 // Skip one more bit_convert if necessary 5569 if (V.getOpcode() == ISD::BITCAST) 5570 V = V.getOperand(0); 5571 5572 if (ISD::isNormalLoad(V.getNode())) { 5573 // Is the original load suitable? 5574 LoadSDNode *LN0 = cast<LoadSDNode>(V); 5575 5576 // FIXME: avoid the multi-use bug that is preventing lots of 5577 // of foldings to be detected, this is still wrong of course, but 5578 // give the temporary desired behavior, and if it happens that 5579 // the load has real more uses, during isel it will not fold, and 5580 // will generate poor code. 5581 if (!LN0 || LN0->isVolatile()) // || !LN0->hasOneUse() 5582 return false; 5583 5584 if (!HasShuffleIntoBitcast) 5585 return true; 5586 5587 // If there's a bitcast before the shuffle, check if the load type and 5588 // alignment is valid. 5589 unsigned Align = LN0->getAlignment(); 5590 unsigned NewAlign = 5591 TLI.getTargetData()->getABITypeAlignment( 5592 VT.getTypeForEVT(*DAG.getContext())); 5593 5594 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VT)) 5595 return false; 5596 } 5597 5598 return true; 5599} 5600 5601static 5602SDValue getMOVDDup(SDValue &Op, DebugLoc &dl, SDValue V1, SelectionDAG &DAG) { 5603 EVT VT = Op.getValueType(); 5604 5605 // Canonizalize to v2f64. 5606 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); 5607 return DAG.getNode(ISD::BITCAST, dl, VT, 5608 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64, 5609 V1, DAG)); 5610} 5611 5612static 5613SDValue getMOVLowToHigh(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, 5614 bool HasSSE2) { 5615 SDValue V1 = Op.getOperand(0); 5616 SDValue V2 = Op.getOperand(1); 5617 EVT VT = Op.getValueType(); 5618 5619 assert(VT != MVT::v2i64 && "unsupported shuffle type"); 5620 5621 if (HasSSE2 && VT == MVT::v2f64) 5622 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG); 5623 5624 // v4f32 or v4i32 5625 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V2, DAG); 5626} 5627 5628static 5629SDValue getMOVHighToLow(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG) { 5630 SDValue V1 = Op.getOperand(0); 5631 SDValue V2 = Op.getOperand(1); 5632 EVT VT = Op.getValueType(); 5633 5634 assert((VT == MVT::v4i32 || VT == MVT::v4f32) && 5635 "unsupported shuffle type"); 5636 5637 if (V2.getOpcode() == ISD::UNDEF) 5638 V2 = V1; 5639 5640 // v4i32 or v4f32 5641 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG); 5642} 5643 5644static 5645SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) { 5646 SDValue V1 = Op.getOperand(0); 5647 SDValue V2 = Op.getOperand(1); 5648 EVT VT = Op.getValueType(); 5649 unsigned NumElems = VT.getVectorNumElements(); 5650 5651 // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second 5652 // operand of these instructions is only memory, so check if there's a 5653 // potencial load folding here, otherwise use SHUFPS or MOVSD to match the 5654 // same masks. 5655 bool CanFoldLoad = false; 5656 5657 // Trivial case, when V2 comes from a load. 5658 if (MayFoldVectorLoad(V2)) 5659 CanFoldLoad = true; 5660 5661 // When V1 is a load, it can be folded later into a store in isel, example: 5662 // (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1) 5663 // turns into: 5664 // (MOVLPSmr addr:$src1, VR128:$src2) 5665 // So, recognize this potential and also use MOVLPS or MOVLPD 5666 if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op)) 5667 CanFoldLoad = true; 5668 5669 // Both of them can't be memory operations though. 5670 if (MayFoldVectorLoad(V1) && MayFoldVectorLoad(V2)) 5671 CanFoldLoad = false; 5672 5673 if (CanFoldLoad) { 5674 if (HasSSE2 && NumElems == 2) 5675 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG); 5676 5677 if (NumElems == 4) 5678 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG); 5679 } 5680 5681 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 5682 // movl and movlp will both match v2i64, but v2i64 is never matched by 5683 // movl earlier because we make it strict to avoid messing with the movlp load 5684 // folding logic (see the code above getMOVLP call). Match it here then, 5685 // this is horrible, but will stay like this until we move all shuffle 5686 // matching to x86 specific nodes. Note that for the 1st condition all 5687 // types are matched with movsd. 5688 if ((HasSSE2 && NumElems == 2) || !X86::isMOVLMask(SVOp)) 5689 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); 5690 else if (HasSSE2) 5691 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); 5692 5693 5694 assert(VT != MVT::v4i32 && "unsupported shuffle type"); 5695 5696 // Invert the operand order and use SHUFPS to match it. 5697 return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V2, V1, 5698 X86::getShuffleSHUFImmediate(SVOp), DAG); 5699} 5700 5701static inline unsigned getUNPCKLOpcode(EVT VT, const X86Subtarget *Subtarget) { 5702 switch(VT.getSimpleVT().SimpleTy) { 5703 case MVT::v4i32: return X86ISD::PUNPCKLDQ; 5704 case MVT::v2i64: return X86ISD::PUNPCKLQDQ; 5705 case MVT::v4f32: return X86ISD::UNPCKLPS; 5706 case MVT::v2f64: return X86ISD::UNPCKLPD; 5707 case MVT::v8f32: return X86ISD::VUNPCKLPSY; 5708 case MVT::v4f64: return X86ISD::VUNPCKLPDY; 5709 case MVT::v16i8: return X86ISD::PUNPCKLBW; 5710 case MVT::v8i16: return X86ISD::PUNPCKLWD; 5711 default: 5712 llvm_unreachable("Unknown type for unpckl"); 5713 } 5714 return 0; 5715} 5716 5717static inline unsigned getUNPCKHOpcode(EVT VT) { 5718 switch(VT.getSimpleVT().SimpleTy) { 5719 case MVT::v4i32: return X86ISD::PUNPCKHDQ; 5720 case MVT::v2i64: return X86ISD::PUNPCKHQDQ; 5721 case MVT::v4f32: return X86ISD::UNPCKHPS; 5722 case MVT::v2f64: return X86ISD::UNPCKHPD; 5723 case MVT::v16i8: return X86ISD::PUNPCKHBW; 5724 case MVT::v8i16: return X86ISD::PUNPCKHWD; 5725 default: 5726 llvm_unreachable("Unknown type for unpckh"); 5727 } 5728 return 0; 5729} 5730 5731static 5732SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG, 5733 const TargetLowering &TLI, 5734 const X86Subtarget *Subtarget) { 5735 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 5736 EVT VT = Op.getValueType(); 5737 DebugLoc dl = Op.getDebugLoc(); 5738 SDValue V1 = Op.getOperand(0); 5739 SDValue V2 = Op.getOperand(1); 5740 5741 if (isZeroShuffle(SVOp)) 5742 return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); 5743 5744 // Handle splat operations 5745 if (SVOp->isSplat()) { 5746 unsigned NumElem = VT.getVectorNumElements(); 5747 // Special case, this is the only place now where it's allowed to return 5748 // a vector_shuffle operation without using a target specific node, because 5749 // *hopefully* it will be optimized away by the dag combiner. FIXME: should 5750 // this be moved to DAGCombine instead? 5751 if (NumElem <= 4 && CanXFormVExtractWithShuffleIntoLoad(Op, DAG, TLI)) 5752 return Op; 5753 5754 // Handle splats by matching through known masks 5755 if ((VT.is128BitVector() && NumElem <= 4) || 5756 (VT.is256BitVector() && NumElem <= 8)) 5757 return SDValue(); 5758 5759 // All i16 and i8 vector types can't be used directly by a generic shuffle 5760 // instruction because the target has no such instruction. Generate shuffles 5761 // which repeat i16 and i8 several times until they fit in i32, and then can 5762 // be manipulated by target suported shuffles. After the insertion of the 5763 // necessary shuffles, the result is bitcasted back to v4f32 or v8f32. 5764 return PromoteSplat(SVOp, DAG); 5765 } 5766 5767 // If the shuffle can be profitably rewritten as a narrower shuffle, then 5768 // do it! 5769 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 5770 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 5771 if (NewOp.getNode()) 5772 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp); 5773 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 5774 // FIXME: Figure out a cleaner way to do this. 5775 // Try to make use of movq to zero out the top part. 5776 if (ISD::isBuildVectorAllZeros(V2.getNode())) { 5777 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 5778 if (NewOp.getNode()) { 5779 if (isCommutedMOVL(cast<ShuffleVectorSDNode>(NewOp), true, false)) 5780 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(0), 5781 DAG, Subtarget, dl); 5782 } 5783 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { 5784 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 5785 if (NewOp.getNode() && X86::isMOVLMask(cast<ShuffleVectorSDNode>(NewOp))) 5786 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 5787 DAG, Subtarget, dl); 5788 } 5789 } 5790 return SDValue(); 5791} 5792 5793SDValue 5794X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { 5795 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 5796 SDValue V1 = Op.getOperand(0); 5797 SDValue V2 = Op.getOperand(1); 5798 EVT VT = Op.getValueType(); 5799 DebugLoc dl = Op.getDebugLoc(); 5800 unsigned NumElems = VT.getVectorNumElements(); 5801 bool isMMX = VT.getSizeInBits() == 64; 5802 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 5803 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 5804 bool V1IsSplat = false; 5805 bool V2IsSplat = false; 5806 bool HasSSE2 = Subtarget->hasSSE2() || Subtarget->hasAVX(); 5807 bool HasSSE3 = Subtarget->hasSSE3() || Subtarget->hasAVX(); 5808 bool HasSSSE3 = Subtarget->hasSSSE3() || Subtarget->hasAVX(); 5809 MachineFunction &MF = DAG.getMachineFunction(); 5810 bool OptForSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 5811 5812 // Shuffle operations on MMX not supported. 5813 if (isMMX) 5814 return Op; 5815 5816 // Vector shuffle lowering takes 3 steps: 5817 // 5818 // 1) Normalize the input vectors. Here splats, zeroed vectors, profitable 5819 // narrowing and commutation of operands should be handled. 5820 // 2) Matching of shuffles with known shuffle masks to x86 target specific 5821 // shuffle nodes. 5822 // 3) Rewriting of unmatched masks into new generic shuffle operations, 5823 // so the shuffle can be broken into other shuffles and the legalizer can 5824 // try the lowering again. 5825 // 5826 // The general ideia is that no vector_shuffle operation should be left to 5827 // be matched during isel, all of them must be converted to a target specific 5828 // node here. 5829 5830 // Normalize the input vectors. Here splats, zeroed vectors, profitable 5831 // narrowing and commutation of operands should be handled. The actual code 5832 // doesn't include all of those, work in progress... 5833 SDValue NewOp = NormalizeVectorShuffle(Op, DAG, *this, Subtarget); 5834 if (NewOp.getNode()) 5835 return NewOp; 5836 5837 // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and 5838 // unpckh_undef). Only use pshufd if speed is more important than size. 5839 if (OptForSize && X86::isUNPCKL_v_undef_Mask(SVOp)) 5840 if (VT != MVT::v2i64 && VT != MVT::v2f64) 5841 return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), dl, VT, V1, V1, DAG); 5842 if (OptForSize && X86::isUNPCKH_v_undef_Mask(SVOp)) 5843 if (VT != MVT::v2i64 && VT != MVT::v2f64) 5844 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); 5845 5846 if (X86::isMOVDDUPMask(SVOp) && HasSSE3 && V2IsUndef && 5847 RelaxedMayFoldVectorLoad(V1)) 5848 return getMOVDDup(Op, dl, V1, DAG); 5849 5850 if (X86::isMOVHLPS_v_undef_Mask(SVOp)) 5851 return getMOVHighToLow(Op, dl, DAG); 5852 5853 // Use to match splats 5854 if (HasSSE2 && X86::isUNPCKHMask(SVOp) && V2IsUndef && 5855 (VT == MVT::v2f64 || VT == MVT::v2i64)) 5856 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); 5857 5858 if (X86::isPSHUFDMask(SVOp)) { 5859 // The actual implementation will match the mask in the if above and then 5860 // during isel it can match several different instructions, not only pshufd 5861 // as its name says, sad but true, emulate the behavior for now... 5862 if (X86::isMOVDDUPMask(SVOp) && ((VT == MVT::v4f32 || VT == MVT::v2i64))) 5863 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG); 5864 5865 unsigned TargetMask = X86::getShuffleSHUFImmediate(SVOp); 5866 5867 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32)) 5868 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG); 5869 5870 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64)) 5871 return getTargetShuffleNode(X86ISD::SHUFPD, dl, VT, V1, V1, 5872 TargetMask, DAG); 5873 5874 if (VT == MVT::v4f32) 5875 return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V1, V1, 5876 TargetMask, DAG); 5877 } 5878 5879 // Check if this can be converted into a logical shift. 5880 bool isLeft = false; 5881 unsigned ShAmt = 0; 5882 SDValue ShVal; 5883 bool isShift = getSubtarget()->hasSSE2() && 5884 isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt); 5885 if (isShift && ShVal.hasOneUse()) { 5886 // If the shifted value has multiple uses, it may be cheaper to use 5887 // v_set0 + movlhps or movhlps, etc. 5888 EVT EltVT = VT.getVectorElementType(); 5889 ShAmt *= EltVT.getSizeInBits(); 5890 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 5891 } 5892 5893 if (X86::isMOVLMask(SVOp)) { 5894 if (V1IsUndef) 5895 return V2; 5896 if (ISD::isBuildVectorAllZeros(V1.getNode())) 5897 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl); 5898 if (!X86::isMOVLPMask(SVOp)) { 5899 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64)) 5900 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); 5901 5902 if (VT == MVT::v4i32 || VT == MVT::v4f32) 5903 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); 5904 } 5905 } 5906 5907 // FIXME: fold these into legal mask. 5908 if (X86::isMOVLHPSMask(SVOp) && !X86::isUNPCKLMask(SVOp)) 5909 return getMOVLowToHigh(Op, dl, DAG, HasSSE2); 5910 5911 if (X86::isMOVHLPSMask(SVOp)) 5912 return getMOVHighToLow(Op, dl, DAG); 5913 5914 if (X86::isMOVSHDUPMask(SVOp) && HasSSE3 && V2IsUndef && NumElems == 4) 5915 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG); 5916 5917 if (X86::isMOVSLDUPMask(SVOp) && HasSSE3 && V2IsUndef && NumElems == 4) 5918 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG); 5919 5920 if (X86::isMOVLPMask(SVOp)) 5921 return getMOVLP(Op, dl, DAG, HasSSE2); 5922 5923 if (ShouldXformToMOVHLPS(SVOp) || 5924 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), SVOp)) 5925 return CommuteVectorShuffle(SVOp, DAG); 5926 5927 if (isShift) { 5928 // No better options. Use a vshl / vsrl. 5929 EVT EltVT = VT.getVectorElementType(); 5930 ShAmt *= EltVT.getSizeInBits(); 5931 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 5932 } 5933 5934 bool Commuted = false; 5935 // FIXME: This should also accept a bitcast of a splat? Be careful, not 5936 // 1,1,1,1 -> v8i16 though. 5937 V1IsSplat = isSplatVector(V1.getNode()); 5938 V2IsSplat = isSplatVector(V2.getNode()); 5939 5940 // Canonicalize the splat or undef, if present, to be on the RHS. 5941 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 5942 Op = CommuteVectorShuffle(SVOp, DAG); 5943 SVOp = cast<ShuffleVectorSDNode>(Op); 5944 V1 = SVOp->getOperand(0); 5945 V2 = SVOp->getOperand(1); 5946 std::swap(V1IsSplat, V2IsSplat); 5947 std::swap(V1IsUndef, V2IsUndef); 5948 Commuted = true; 5949 } 5950 5951 if (isCommutedMOVL(SVOp, V2IsSplat, V2IsUndef)) { 5952 // Shuffling low element of v1 into undef, just return v1. 5953 if (V2IsUndef) 5954 return V1; 5955 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which 5956 // the instruction selector will not match, so get a canonical MOVL with 5957 // swapped operands to undo the commute. 5958 return getMOVL(DAG, dl, VT, V2, V1); 5959 } 5960 5961 if (X86::isUNPCKLMask(SVOp)) 5962 return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), 5963 dl, VT, V1, V2, DAG); 5964 5965 if (X86::isUNPCKHMask(SVOp)) 5966 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V2, DAG); 5967 5968 if (V2IsSplat) { 5969 // Normalize mask so all entries that point to V2 points to its first 5970 // element then try to match unpck{h|l} again. If match, return a 5971 // new vector_shuffle with the corrected mask. 5972 SDValue NewMask = NormalizeMask(SVOp, DAG); 5973 ShuffleVectorSDNode *NSVOp = cast<ShuffleVectorSDNode>(NewMask); 5974 if (NSVOp != SVOp) { 5975 if (X86::isUNPCKLMask(NSVOp, true)) { 5976 return NewMask; 5977 } else if (X86::isUNPCKHMask(NSVOp, true)) { 5978 return NewMask; 5979 } 5980 } 5981 } 5982 5983 if (Commuted) { 5984 // Commute is back and try unpck* again. 5985 // FIXME: this seems wrong. 5986 SDValue NewOp = CommuteVectorShuffle(SVOp, DAG); 5987 ShuffleVectorSDNode *NewSVOp = cast<ShuffleVectorSDNode>(NewOp); 5988 5989 if (X86::isUNPCKLMask(NewSVOp)) 5990 return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), 5991 dl, VT, V2, V1, DAG); 5992 5993 if (X86::isUNPCKHMask(NewSVOp)) 5994 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V2, V1, DAG); 5995 } 5996 5997 // Normalize the node to match x86 shuffle ops if needed 5998 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(SVOp)) 5999 return CommuteVectorShuffle(SVOp, DAG); 6000 6001 // The checks below are all present in isShuffleMaskLegal, but they are 6002 // inlined here right now to enable us to directly emit target specific 6003 // nodes, and remove one by one until they don't return Op anymore. 6004 SmallVector<int, 16> M; 6005 SVOp->getMask(M); 6006 6007 if (isPALIGNRMask(M, VT, HasSSSE3)) 6008 return getTargetShuffleNode(X86ISD::PALIGN, dl, VT, V1, V2, 6009 X86::getShufflePALIGNRImmediate(SVOp), 6010 DAG); 6011 6012 if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) && 6013 SVOp->getSplatIndex() == 0 && V2IsUndef) { 6014 if (VT == MVT::v2f64) 6015 return getTargetShuffleNode(X86ISD::UNPCKLPD, dl, VT, V1, V1, DAG); 6016 if (VT == MVT::v2i64) 6017 return getTargetShuffleNode(X86ISD::PUNPCKLQDQ, dl, VT, V1, V1, DAG); 6018 } 6019 6020 if (isPSHUFHWMask(M, VT)) 6021 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1, 6022 X86::getShufflePSHUFHWImmediate(SVOp), 6023 DAG); 6024 6025 if (isPSHUFLWMask(M, VT)) 6026 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1, 6027 X86::getShufflePSHUFLWImmediate(SVOp), 6028 DAG); 6029 6030 if (isSHUFPMask(M, VT)) { 6031 unsigned TargetMask = X86::getShuffleSHUFImmediate(SVOp); 6032 if (VT == MVT::v4f32 || VT == MVT::v4i32) 6033 return getTargetShuffleNode(X86ISD::SHUFPS, dl, VT, V1, V2, 6034 TargetMask, DAG); 6035 if (VT == MVT::v2f64 || VT == MVT::v2i64) 6036 return getTargetShuffleNode(X86ISD::SHUFPD, dl, VT, V1, V2, 6037 TargetMask, DAG); 6038 } 6039 6040 if (X86::isUNPCKL_v_undef_Mask(SVOp)) 6041 if (VT != MVT::v2i64 && VT != MVT::v2f64) 6042 return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), 6043 dl, VT, V1, V1, DAG); 6044 if (X86::isUNPCKH_v_undef_Mask(SVOp)) 6045 if (VT != MVT::v2i64 && VT != MVT::v2f64) 6046 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); 6047 6048 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 6049 if (VT == MVT::v8i16) { 6050 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, DAG); 6051 if (NewOp.getNode()) 6052 return NewOp; 6053 } 6054 6055 if (VT == MVT::v16i8) { 6056 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this); 6057 if (NewOp.getNode()) 6058 return NewOp; 6059 } 6060 6061 // Handle all 128-bit wide vectors with 4 elements, and match them with 6062 // several different shuffle types. 6063 if (NumElems == 4 && VT.getSizeInBits() == 128) 6064 return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG); 6065 6066 //===--------------------------------------------------------------------===// 6067 // Custom lower or generate target specific nodes for 256-bit shuffles. 6068 6069 // Handle VPERMIL permutations 6070 if (isVPERMILMask(M, VT)) { 6071 unsigned TargetMask = getShuffleVPERMILImmediate(SVOp); 6072 if (VT == MVT::v8f32) 6073 return getTargetShuffleNode(X86ISD::VPERMIL, dl, VT, V1, TargetMask, DAG); 6074 } 6075 6076 // Handle general 256-bit shuffles 6077 if (VT.is256BitVector()) 6078 return LowerVECTOR_SHUFFLE_256(SVOp, DAG); 6079 6080 return SDValue(); 6081} 6082 6083SDValue 6084X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 6085 SelectionDAG &DAG) const { 6086 EVT VT = Op.getValueType(); 6087 DebugLoc dl = Op.getDebugLoc(); 6088 if (VT.getSizeInBits() == 8) { 6089 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, 6090 Op.getOperand(0), Op.getOperand(1)); 6091 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 6092 DAG.getValueType(VT)); 6093 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6094 } else if (VT.getSizeInBits() == 16) { 6095 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6096 // If Idx is 0, it's cheaper to do a move instead of a pextrw. 6097 if (Idx == 0) 6098 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 6099 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6100 DAG.getNode(ISD::BITCAST, dl, 6101 MVT::v4i32, 6102 Op.getOperand(0)), 6103 Op.getOperand(1))); 6104 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, 6105 Op.getOperand(0), Op.getOperand(1)); 6106 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 6107 DAG.getValueType(VT)); 6108 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6109 } else if (VT == MVT::f32) { 6110 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 6111 // the result back to FR32 register. It's only worth matching if the 6112 // result has a single use which is a store or a bitcast to i32. And in 6113 // the case of a store, it's not worth it if the index is a constant 0, 6114 // because a MOVSSmr can be used instead, which is smaller and faster. 6115 if (!Op.hasOneUse()) 6116 return SDValue(); 6117 SDNode *User = *Op.getNode()->use_begin(); 6118 if ((User->getOpcode() != ISD::STORE || 6119 (isa<ConstantSDNode>(Op.getOperand(1)) && 6120 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) && 6121 (User->getOpcode() != ISD::BITCAST || 6122 User->getValueType(0) != MVT::i32)) 6123 return SDValue(); 6124 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6125 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, 6126 Op.getOperand(0)), 6127 Op.getOperand(1)); 6128 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); 6129 } else if (VT == MVT::i32) { 6130 // ExtractPS works with constant index. 6131 if (isa<ConstantSDNode>(Op.getOperand(1))) 6132 return Op; 6133 } 6134 return SDValue(); 6135} 6136 6137 6138SDValue 6139X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 6140 SelectionDAG &DAG) const { 6141 if (!isa<ConstantSDNode>(Op.getOperand(1))) 6142 return SDValue(); 6143 6144 SDValue Vec = Op.getOperand(0); 6145 EVT VecVT = Vec.getValueType(); 6146 6147 // If this is a 256-bit vector result, first extract the 128-bit 6148 // vector and then extract from the 128-bit vector. 6149 if (VecVT.getSizeInBits() > 128) { 6150 DebugLoc dl = Op.getNode()->getDebugLoc(); 6151 unsigned NumElems = VecVT.getVectorNumElements(); 6152 SDValue Idx = Op.getOperand(1); 6153 6154 if (!isa<ConstantSDNode>(Idx)) 6155 return SDValue(); 6156 6157 unsigned ExtractNumElems = NumElems / (VecVT.getSizeInBits() / 128); 6158 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 6159 6160 // Get the 128-bit vector. 6161 bool Upper = IdxVal >= ExtractNumElems; 6162 Vec = Extract128BitVector(Vec, Idx, DAG, dl); 6163 6164 // Extract from it. 6165 SDValue ScaledIdx = Idx; 6166 if (Upper) 6167 ScaledIdx = DAG.getNode(ISD::SUB, dl, Idx.getValueType(), Idx, 6168 DAG.getConstant(ExtractNumElems, 6169 Idx.getValueType())); 6170 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, 6171 ScaledIdx); 6172 } 6173 6174 assert(Vec.getValueSizeInBits() <= 128 && "Unexpected vector length"); 6175 6176 if (Subtarget->hasSSE41()) { 6177 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 6178 if (Res.getNode()) 6179 return Res; 6180 } 6181 6182 EVT VT = Op.getValueType(); 6183 DebugLoc dl = Op.getDebugLoc(); 6184 // TODO: handle v16i8. 6185 if (VT.getSizeInBits() == 16) { 6186 SDValue Vec = Op.getOperand(0); 6187 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6188 if (Idx == 0) 6189 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 6190 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6191 DAG.getNode(ISD::BITCAST, dl, 6192 MVT::v4i32, Vec), 6193 Op.getOperand(1))); 6194 // Transform it so it match pextrw which produces a 32-bit result. 6195 EVT EltVT = MVT::i32; 6196 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT, 6197 Op.getOperand(0), Op.getOperand(1)); 6198 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract, 6199 DAG.getValueType(VT)); 6200 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6201 } else if (VT.getSizeInBits() == 32) { 6202 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6203 if (Idx == 0) 6204 return Op; 6205 6206 // SHUFPS the element to the lowest double word, then movss. 6207 int Mask[4] = { Idx, -1, -1, -1 }; 6208 EVT VVT = Op.getOperand(0).getValueType(); 6209 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 6210 DAG.getUNDEF(VVT), Mask); 6211 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 6212 DAG.getIntPtrConstant(0)); 6213 } else if (VT.getSizeInBits() == 64) { 6214 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 6215 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 6216 // to match extract_elt for f64. 6217 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6218 if (Idx == 0) 6219 return Op; 6220 6221 // UNPCKHPD the element to the lowest double word, then movsd. 6222 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 6223 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 6224 int Mask[2] = { 1, -1 }; 6225 EVT VVT = Op.getOperand(0).getValueType(); 6226 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 6227 DAG.getUNDEF(VVT), Mask); 6228 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 6229 DAG.getIntPtrConstant(0)); 6230 } 6231 6232 return SDValue(); 6233} 6234 6235SDValue 6236X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, 6237 SelectionDAG &DAG) const { 6238 EVT VT = Op.getValueType(); 6239 EVT EltVT = VT.getVectorElementType(); 6240 DebugLoc dl = Op.getDebugLoc(); 6241 6242 SDValue N0 = Op.getOperand(0); 6243 SDValue N1 = Op.getOperand(1); 6244 SDValue N2 = Op.getOperand(2); 6245 6246 if ((EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) && 6247 isa<ConstantSDNode>(N2)) { 6248 unsigned Opc; 6249 if (VT == MVT::v8i16) 6250 Opc = X86ISD::PINSRW; 6251 else if (VT == MVT::v16i8) 6252 Opc = X86ISD::PINSRB; 6253 else 6254 Opc = X86ISD::PINSRB; 6255 6256 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 6257 // argument. 6258 if (N1.getValueType() != MVT::i32) 6259 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 6260 if (N2.getValueType() != MVT::i32) 6261 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 6262 return DAG.getNode(Opc, dl, VT, N0, N1, N2); 6263 } else if (EltVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 6264 // Bits [7:6] of the constant are the source select. This will always be 6265 // zero here. The DAG Combiner may combine an extract_elt index into these 6266 // bits. For example (insert (extract, 3), 2) could be matched by putting 6267 // the '3' into bits [7:6] of X86ISD::INSERTPS. 6268 // Bits [5:4] of the constant are the destination select. This is the 6269 // value of the incoming immediate. 6270 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 6271 // combine either bitwise AND or insert of float 0.0 to set these bits. 6272 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4); 6273 // Create this as a scalar to vector.. 6274 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); 6275 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2); 6276 } else if (EltVT == MVT::i32 && isa<ConstantSDNode>(N2)) { 6277 // PINSR* works with constant index. 6278 return Op; 6279 } 6280 return SDValue(); 6281} 6282 6283SDValue 6284X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { 6285 EVT VT = Op.getValueType(); 6286 EVT EltVT = VT.getVectorElementType(); 6287 6288 DebugLoc dl = Op.getDebugLoc(); 6289 SDValue N0 = Op.getOperand(0); 6290 SDValue N1 = Op.getOperand(1); 6291 SDValue N2 = Op.getOperand(2); 6292 6293 // If this is a 256-bit vector result, first insert into a 128-bit 6294 // vector and then insert into the 256-bit vector. 6295 if (VT.getSizeInBits() > 128) { 6296 if (!isa<ConstantSDNode>(N2)) 6297 return SDValue(); 6298 6299 // Get the 128-bit vector. 6300 unsigned NumElems = VT.getVectorNumElements(); 6301 unsigned IdxVal = cast<ConstantSDNode>(N2)->getZExtValue(); 6302 bool Upper = IdxVal >= NumElems / 2; 6303 6304 SDValue SubN0 = Extract128BitVector(N0, N2, DAG, dl); 6305 6306 // Insert into it. 6307 SDValue ScaledN2 = N2; 6308 if (Upper) 6309 ScaledN2 = DAG.getNode(ISD::SUB, dl, N2.getValueType(), N2, 6310 DAG.getConstant(NumElems / 6311 (VT.getSizeInBits() / 128), 6312 N2.getValueType())); 6313 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubN0.getValueType(), SubN0, 6314 N1, ScaledN2); 6315 6316 // Insert the 128-bit vector 6317 // FIXME: Why UNDEF? 6318 return Insert128BitVector(N0, Op, N2, DAG, dl); 6319 } 6320 6321 if (Subtarget->hasSSE41()) 6322 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 6323 6324 if (EltVT == MVT::i8) 6325 return SDValue(); 6326 6327 if (EltVT.getSizeInBits() == 16 && isa<ConstantSDNode>(N2)) { 6328 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 6329 // as its second argument. 6330 if (N1.getValueType() != MVT::i32) 6331 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 6332 if (N2.getValueType() != MVT::i32) 6333 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 6334 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2); 6335 } 6336 return SDValue(); 6337} 6338 6339SDValue 6340X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const { 6341 DebugLoc dl = Op.getDebugLoc(); 6342 EVT OpVT = Op.getValueType(); 6343 6344 if (Op.getValueType() == MVT::v1i64 && 6345 Op.getOperand(0).getValueType() == MVT::i64) 6346 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); 6347 6348 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); 6349 assert(Op.getValueType().getSimpleVT().getSizeInBits() == 128 && 6350 "Expected an SSE type!"); 6351 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), 6352 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); 6353} 6354 6355// Lower a node with an EXTRACT_SUBVECTOR opcode. This may result in 6356// a simple subregister reference or explicit instructions to grab 6357// upper bits of a vector. 6358SDValue 6359X86TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { 6360 if (Subtarget->hasAVX()) { 6361 DebugLoc dl = Op.getNode()->getDebugLoc(); 6362 SDValue Vec = Op.getNode()->getOperand(0); 6363 SDValue Idx = Op.getNode()->getOperand(1); 6364 6365 if (Op.getNode()->getValueType(0).getSizeInBits() == 128 6366 && Vec.getNode()->getValueType(0).getSizeInBits() == 256) { 6367 return Extract128BitVector(Vec, Idx, DAG, dl); 6368 } 6369 } 6370 return SDValue(); 6371} 6372 6373// Lower a node with an INSERT_SUBVECTOR opcode. This may result in a 6374// simple superregister reference or explicit instructions to insert 6375// the upper bits of a vector. 6376SDValue 6377X86TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { 6378 if (Subtarget->hasAVX()) { 6379 DebugLoc dl = Op.getNode()->getDebugLoc(); 6380 SDValue Vec = Op.getNode()->getOperand(0); 6381 SDValue SubVec = Op.getNode()->getOperand(1); 6382 SDValue Idx = Op.getNode()->getOperand(2); 6383 6384 if (Op.getNode()->getValueType(0).getSizeInBits() == 256 6385 && SubVec.getNode()->getValueType(0).getSizeInBits() == 128) { 6386 return Insert128BitVector(Vec, SubVec, Idx, DAG, dl); 6387 } 6388 } 6389 return SDValue(); 6390} 6391 6392// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 6393// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 6394// one of the above mentioned nodes. It has to be wrapped because otherwise 6395// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 6396// be used to form addressing mode. These wrapped nodes will be selected 6397// into MOV32ri. 6398SDValue 6399X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { 6400 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 6401 6402 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 6403 // global base reg. 6404 unsigned char OpFlag = 0; 6405 unsigned WrapperKind = X86ISD::Wrapper; 6406 CodeModel::Model M = getTargetMachine().getCodeModel(); 6407 6408 if (Subtarget->isPICStyleRIPRel() && 6409 (M == CodeModel::Small || M == CodeModel::Kernel)) 6410 WrapperKind = X86ISD::WrapperRIP; 6411 else if (Subtarget->isPICStyleGOT()) 6412 OpFlag = X86II::MO_GOTOFF; 6413 else if (Subtarget->isPICStyleStubPIC()) 6414 OpFlag = X86II::MO_PIC_BASE_OFFSET; 6415 6416 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(), 6417 CP->getAlignment(), 6418 CP->getOffset(), OpFlag); 6419 DebugLoc DL = CP->getDebugLoc(); 6420 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 6421 // With PIC, the address is actually $g + Offset. 6422 if (OpFlag) { 6423 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 6424 DAG.getNode(X86ISD::GlobalBaseReg, 6425 DebugLoc(), getPointerTy()), 6426 Result); 6427 } 6428 6429 return Result; 6430} 6431 6432SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 6433 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 6434 6435 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 6436 // global base reg. 6437 unsigned char OpFlag = 0; 6438 unsigned WrapperKind = X86ISD::Wrapper; 6439 CodeModel::Model M = getTargetMachine().getCodeModel(); 6440 6441 if (Subtarget->isPICStyleRIPRel() && 6442 (M == CodeModel::Small || M == CodeModel::Kernel)) 6443 WrapperKind = X86ISD::WrapperRIP; 6444 else if (Subtarget->isPICStyleGOT()) 6445 OpFlag = X86II::MO_GOTOFF; 6446 else if (Subtarget->isPICStyleStubPIC()) 6447 OpFlag = X86II::MO_PIC_BASE_OFFSET; 6448 6449 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(), 6450 OpFlag); 6451 DebugLoc DL = JT->getDebugLoc(); 6452 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 6453 6454 // With PIC, the address is actually $g + Offset. 6455 if (OpFlag) 6456 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 6457 DAG.getNode(X86ISD::GlobalBaseReg, 6458 DebugLoc(), getPointerTy()), 6459 Result); 6460 6461 return Result; 6462} 6463 6464SDValue 6465X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const { 6466 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 6467 6468 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 6469 // global base reg. 6470 unsigned char OpFlag = 0; 6471 unsigned WrapperKind = X86ISD::Wrapper; 6472 CodeModel::Model M = getTargetMachine().getCodeModel(); 6473 6474 if (Subtarget->isPICStyleRIPRel() && 6475 (M == CodeModel::Small || M == CodeModel::Kernel)) 6476 WrapperKind = X86ISD::WrapperRIP; 6477 else if (Subtarget->isPICStyleGOT()) 6478 OpFlag = X86II::MO_GOTOFF; 6479 else if (Subtarget->isPICStyleStubPIC()) 6480 OpFlag = X86II::MO_PIC_BASE_OFFSET; 6481 6482 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag); 6483 6484 DebugLoc DL = Op.getDebugLoc(); 6485 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 6486 6487 6488 // With PIC, the address is actually $g + Offset. 6489 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 6490 !Subtarget->is64Bit()) { 6491 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 6492 DAG.getNode(X86ISD::GlobalBaseReg, 6493 DebugLoc(), getPointerTy()), 6494 Result); 6495 } 6496 6497 return Result; 6498} 6499 6500SDValue 6501X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { 6502 // Create the TargetBlockAddressAddress node. 6503 unsigned char OpFlags = 6504 Subtarget->ClassifyBlockAddressReference(); 6505 CodeModel::Model M = getTargetMachine().getCodeModel(); 6506 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 6507 DebugLoc dl = Op.getDebugLoc(); 6508 SDValue Result = DAG.getBlockAddress(BA, getPointerTy(), 6509 /*isTarget=*/true, OpFlags); 6510 6511 if (Subtarget->isPICStyleRIPRel() && 6512 (M == CodeModel::Small || M == CodeModel::Kernel)) 6513 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); 6514 else 6515 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); 6516 6517 // With PIC, the address is actually $g + Offset. 6518 if (isGlobalRelativeToPICBase(OpFlags)) { 6519 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), 6520 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), 6521 Result); 6522 } 6523 6524 return Result; 6525} 6526 6527SDValue 6528X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl, 6529 int64_t Offset, 6530 SelectionDAG &DAG) const { 6531 // Create the TargetGlobalAddress node, folding in the constant 6532 // offset if it is legal. 6533 unsigned char OpFlags = 6534 Subtarget->ClassifyGlobalReference(GV, getTargetMachine()); 6535 CodeModel::Model M = getTargetMachine().getCodeModel(); 6536 SDValue Result; 6537 if (OpFlags == X86II::MO_NO_FLAG && 6538 X86::isOffsetSuitableForCodeModel(Offset, M)) { 6539 // A direct static reference to a global. 6540 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset); 6541 Offset = 0; 6542 } else { 6543 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags); 6544 } 6545 6546 if (Subtarget->isPICStyleRIPRel() && 6547 (M == CodeModel::Small || M == CodeModel::Kernel)) 6548 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); 6549 else 6550 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); 6551 6552 // With PIC, the address is actually $g + Offset. 6553 if (isGlobalRelativeToPICBase(OpFlags)) { 6554 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), 6555 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), 6556 Result); 6557 } 6558 6559 // For globals that require a load from a stub to get the address, emit the 6560 // load. 6561 if (isGlobalStubReference(OpFlags)) 6562 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result, 6563 MachinePointerInfo::getGOT(), false, false, 0); 6564 6565 // If there was a non-zero offset that we didn't fold, create an explicit 6566 // addition for it. 6567 if (Offset != 0) 6568 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result, 6569 DAG.getConstant(Offset, getPointerTy())); 6570 6571 return Result; 6572} 6573 6574SDValue 6575X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { 6576 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 6577 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset(); 6578 return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG); 6579} 6580 6581static SDValue 6582GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, 6583 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, 6584 unsigned char OperandFlags) { 6585 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6586 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6587 DebugLoc dl = GA->getDebugLoc(); 6588 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, 6589 GA->getValueType(0), 6590 GA->getOffset(), 6591 OperandFlags); 6592 if (InFlag) { 6593 SDValue Ops[] = { Chain, TGA, *InFlag }; 6594 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3); 6595 } else { 6596 SDValue Ops[] = { Chain, TGA }; 6597 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2); 6598 } 6599 6600 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls. 6601 MFI->setAdjustsStack(true); 6602 6603 SDValue Flag = Chain.getValue(1); 6604 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag); 6605} 6606 6607// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 6608static SDValue 6609LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 6610 const EVT PtrVT) { 6611 SDValue InFlag; 6612 DebugLoc dl = GA->getDebugLoc(); // ? function entry point might be better 6613 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, 6614 DAG.getNode(X86ISD::GlobalBaseReg, 6615 DebugLoc(), PtrVT), InFlag); 6616 InFlag = Chain.getValue(1); 6617 6618 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD); 6619} 6620 6621// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 6622static SDValue 6623LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 6624 const EVT PtrVT) { 6625 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT, 6626 X86::RAX, X86II::MO_TLSGD); 6627} 6628 6629// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 6630// "local exec" model. 6631static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 6632 const EVT PtrVT, TLSModel::Model model, 6633 bool is64Bit) { 6634 DebugLoc dl = GA->getDebugLoc(); 6635 6636 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit). 6637 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(), 6638 is64Bit ? 257 : 256)); 6639 6640 SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 6641 DAG.getIntPtrConstant(0), 6642 MachinePointerInfo(Ptr), false, false, 0); 6643 6644 unsigned char OperandFlags = 0; 6645 // Most TLS accesses are not RIP relative, even on x86-64. One exception is 6646 // initialexec. 6647 unsigned WrapperKind = X86ISD::Wrapper; 6648 if (model == TLSModel::LocalExec) { 6649 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF; 6650 } else if (is64Bit) { 6651 assert(model == TLSModel::InitialExec); 6652 OperandFlags = X86II::MO_GOTTPOFF; 6653 WrapperKind = X86ISD::WrapperRIP; 6654 } else { 6655 assert(model == TLSModel::InitialExec); 6656 OperandFlags = X86II::MO_INDNTPOFF; 6657 } 6658 6659 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 6660 // exec) 6661 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, 6662 GA->getValueType(0), 6663 GA->getOffset(), OperandFlags); 6664 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); 6665 6666 if (model == TLSModel::InitialExec) 6667 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset, 6668 MachinePointerInfo::getGOT(), false, false, 0); 6669 6670 // The address of the thread local variable is the add of the thread 6671 // pointer with the offset of the variable. 6672 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); 6673} 6674 6675SDValue 6676X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { 6677 6678 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 6679 const GlobalValue *GV = GA->getGlobal(); 6680 6681 if (Subtarget->isTargetELF()) { 6682 // TODO: implement the "local dynamic" model 6683 // TODO: implement the "initial exec"model for pic executables 6684 6685 // If GV is an alias then use the aliasee for determining 6686 // thread-localness. 6687 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 6688 GV = GA->resolveAliasedGlobal(false); 6689 6690 TLSModel::Model model 6691 = getTLSModel(GV, getTargetMachine().getRelocationModel()); 6692 6693 switch (model) { 6694 case TLSModel::GeneralDynamic: 6695 case TLSModel::LocalDynamic: // not implemented 6696 if (Subtarget->is64Bit()) 6697 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 6698 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 6699 6700 case TLSModel::InitialExec: 6701 case TLSModel::LocalExec: 6702 return LowerToTLSExecModel(GA, DAG, getPointerTy(), model, 6703 Subtarget->is64Bit()); 6704 } 6705 } else if (Subtarget->isTargetDarwin()) { 6706 // Darwin only has one model of TLS. Lower to that. 6707 unsigned char OpFlag = 0; 6708 unsigned WrapperKind = Subtarget->isPICStyleRIPRel() ? 6709 X86ISD::WrapperRIP : X86ISD::Wrapper; 6710 6711 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 6712 // global base reg. 6713 bool PIC32 = (getTargetMachine().getRelocationModel() == Reloc::PIC_) && 6714 !Subtarget->is64Bit(); 6715 if (PIC32) 6716 OpFlag = X86II::MO_TLVP_PIC_BASE; 6717 else 6718 OpFlag = X86II::MO_TLVP; 6719 DebugLoc DL = Op.getDebugLoc(); 6720 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, 6721 GA->getValueType(0), 6722 GA->getOffset(), OpFlag); 6723 SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 6724 6725 // With PIC32, the address is actually $g + Offset. 6726 if (PIC32) 6727 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(), 6728 DAG.getNode(X86ISD::GlobalBaseReg, 6729 DebugLoc(), getPointerTy()), 6730 Offset); 6731 6732 // Lowering the machine isd will make sure everything is in the right 6733 // location. 6734 SDValue Chain = DAG.getEntryNode(); 6735 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6736 SDValue Args[] = { Chain, Offset }; 6737 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args, 2); 6738 6739 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls. 6740 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6741 MFI->setAdjustsStack(true); 6742 6743 // And our return value (tls address) is in the standard call return value 6744 // location. 6745 unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; 6746 return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy()); 6747 } 6748 6749 assert(false && 6750 "TLS not implemented for this target."); 6751 6752 llvm_unreachable("Unreachable"); 6753 return SDValue(); 6754} 6755 6756 6757/// LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values and 6758/// take a 2 x i32 value to shift plus a shift amount. 6759SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const { 6760 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 6761 EVT VT = Op.getValueType(); 6762 unsigned VTBits = VT.getSizeInBits(); 6763 DebugLoc dl = Op.getDebugLoc(); 6764 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 6765 SDValue ShOpLo = Op.getOperand(0); 6766 SDValue ShOpHi = Op.getOperand(1); 6767 SDValue ShAmt = Op.getOperand(2); 6768 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, 6769 DAG.getConstant(VTBits - 1, MVT::i8)) 6770 : DAG.getConstant(0, VT); 6771 6772 SDValue Tmp2, Tmp3; 6773 if (Op.getOpcode() == ISD::SHL_PARTS) { 6774 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt); 6775 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 6776 } else { 6777 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt); 6778 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt); 6779 } 6780 6781 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, 6782 DAG.getConstant(VTBits, MVT::i8)); 6783 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, 6784 AndNode, DAG.getConstant(0, MVT::i8)); 6785 6786 SDValue Hi, Lo; 6787 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 6788 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 6789 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 6790 6791 if (Op.getOpcode() == ISD::SHL_PARTS) { 6792 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 6793 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 6794 } else { 6795 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 6796 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 6797 } 6798 6799 SDValue Ops[2] = { Lo, Hi }; 6800 return DAG.getMergeValues(Ops, 2, dl); 6801} 6802 6803SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, 6804 SelectionDAG &DAG) const { 6805 EVT SrcVT = Op.getOperand(0).getValueType(); 6806 6807 if (SrcVT.isVector()) 6808 return SDValue(); 6809 6810 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 6811 "Unknown SINT_TO_FP to lower!"); 6812 6813 // These are really Legal; return the operand so the caller accepts it as 6814 // Legal. 6815 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 6816 return Op; 6817 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && 6818 Subtarget->is64Bit()) { 6819 return Op; 6820 } 6821 6822 DebugLoc dl = Op.getDebugLoc(); 6823 unsigned Size = SrcVT.getSizeInBits()/8; 6824 MachineFunction &MF = DAG.getMachineFunction(); 6825 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false); 6826 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 6827 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 6828 StackSlot, 6829 MachinePointerInfo::getFixedStack(SSFI), 6830 false, false, 0); 6831 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG); 6832} 6833 6834SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, 6835 SDValue StackSlot, 6836 SelectionDAG &DAG) const { 6837 // Build the FILD 6838 DebugLoc DL = Op.getDebugLoc(); 6839 SDVTList Tys; 6840 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 6841 if (useSSE) 6842 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); 6843 else 6844 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 6845 6846 unsigned ByteSize = SrcVT.getSizeInBits()/8; 6847 6848 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(StackSlot); 6849 MachineMemOperand *MMO; 6850 if (FI) { 6851 int SSFI = FI->getIndex(); 6852 MMO = 6853 DAG.getMachineFunction() 6854 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 6855 MachineMemOperand::MOLoad, ByteSize, ByteSize); 6856 } else { 6857 MMO = cast<LoadSDNode>(StackSlot)->getMemOperand(); 6858 StackSlot = StackSlot.getOperand(1); 6859 } 6860 SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) }; 6861 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG : 6862 X86ISD::FILD, DL, 6863 Tys, Ops, array_lengthof(Ops), 6864 SrcVT, MMO); 6865 6866 if (useSSE) { 6867 Chain = Result.getValue(1); 6868 SDValue InFlag = Result.getValue(2); 6869 6870 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 6871 // shouldn't be necessary except that RFP cannot be live across 6872 // multiple blocks. When stackifier is fixed, they can be uncoupled. 6873 MachineFunction &MF = DAG.getMachineFunction(); 6874 unsigned SSFISize = Op.getValueType().getSizeInBits()/8; 6875 int SSFI = MF.getFrameInfo()->CreateStackObject(SSFISize, SSFISize, false); 6876 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 6877 Tys = DAG.getVTList(MVT::Other); 6878 SDValue Ops[] = { 6879 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag 6880 }; 6881 MachineMemOperand *MMO = 6882 DAG.getMachineFunction() 6883 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 6884 MachineMemOperand::MOStore, SSFISize, SSFISize); 6885 6886 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, 6887 Ops, array_lengthof(Ops), 6888 Op.getValueType(), MMO); 6889 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot, 6890 MachinePointerInfo::getFixedStack(SSFI), 6891 false, false, 0); 6892 } 6893 6894 return Result; 6895} 6896 6897// LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion. 6898SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op, 6899 SelectionDAG &DAG) const { 6900 // This algorithm is not obvious. Here it is in C code, more or less: 6901 /* 6902 double uint64_to_double( uint32_t hi, uint32_t lo ) { 6903 static const __m128i exp = { 0x4330000045300000ULL, 0 }; 6904 static const __m128d bias = { 0x1.0p84, 0x1.0p52 }; 6905 6906 // Copy ints to xmm registers. 6907 __m128i xh = _mm_cvtsi32_si128( hi ); 6908 __m128i xl = _mm_cvtsi32_si128( lo ); 6909 6910 // Combine into low half of a single xmm register. 6911 __m128i x = _mm_unpacklo_epi32( xh, xl ); 6912 __m128d d; 6913 double sd; 6914 6915 // Merge in appropriate exponents to give the integer bits the right 6916 // magnitude. 6917 x = _mm_unpacklo_epi32( x, exp ); 6918 6919 // Subtract away the biases to deal with the IEEE-754 double precision 6920 // implicit 1. 6921 d = _mm_sub_pd( (__m128d) x, bias ); 6922 6923 // All conversions up to here are exact. The correctly rounded result is 6924 // calculated using the current rounding mode using the following 6925 // horizontal add. 6926 d = _mm_add_sd( d, _mm_unpackhi_pd( d, d ) ); 6927 _mm_store_sd( &sd, d ); // Because we are returning doubles in XMM, this 6928 // store doesn't really need to be here (except 6929 // maybe to zero the other double) 6930 return sd; 6931 } 6932 */ 6933 6934 DebugLoc dl = Op.getDebugLoc(); 6935 LLVMContext *Context = DAG.getContext(); 6936 6937 // Build some magic constants. 6938 std::vector<Constant*> CV0; 6939 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x45300000))); 6940 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x43300000))); 6941 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0))); 6942 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0))); 6943 Constant *C0 = ConstantVector::get(CV0); 6944 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16); 6945 6946 std::vector<Constant*> CV1; 6947 CV1.push_back( 6948 ConstantFP::get(*Context, APFloat(APInt(64, 0x4530000000000000ULL)))); 6949 CV1.push_back( 6950 ConstantFP::get(*Context, APFloat(APInt(64, 0x4330000000000000ULL)))); 6951 Constant *C1 = ConstantVector::get(CV1); 6952 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16); 6953 6954 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 6955 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 6956 Op.getOperand(0), 6957 DAG.getIntPtrConstant(1))); 6958 SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 6959 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 6960 Op.getOperand(0), 6961 DAG.getIntPtrConstant(0))); 6962 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, XR1, XR2); 6963 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, 6964 MachinePointerInfo::getConstantPool(), 6965 false, false, 16); 6966 SDValue Unpck2 = getUnpackl(DAG, dl, MVT::v4i32, Unpck1, CLod0); 6967 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck2); 6968 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, 6969 MachinePointerInfo::getConstantPool(), 6970 false, false, 16); 6971 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); 6972 6973 // Add the halves; easiest way is to swap them into another reg first. 6974 int ShufMask[2] = { 1, -1 }; 6975 SDValue Shuf = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, 6976 DAG.getUNDEF(MVT::v2f64), ShufMask); 6977 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuf, Sub); 6978 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Add, 6979 DAG.getIntPtrConstant(0)); 6980} 6981 6982// LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion. 6983SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op, 6984 SelectionDAG &DAG) const { 6985 DebugLoc dl = Op.getDebugLoc(); 6986 // FP constant to bias correct the final result. 6987 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), 6988 MVT::f64); 6989 6990 // Load the 32-bit value into an XMM register. 6991 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 6992 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 6993 Op.getOperand(0), 6994 DAG.getIntPtrConstant(0))); 6995 6996 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 6997 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), 6998 DAG.getIntPtrConstant(0)); 6999 7000 // Or the load with the bias. 7001 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, 7002 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 7003 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 7004 MVT::v2f64, Load)), 7005 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 7006 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 7007 MVT::v2f64, Bias))); 7008 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 7009 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), 7010 DAG.getIntPtrConstant(0)); 7011 7012 // Subtract the bias. 7013 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); 7014 7015 // Handle final rounding. 7016 EVT DestVT = Op.getValueType(); 7017 7018 if (DestVT.bitsLT(MVT::f64)) { 7019 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 7020 DAG.getIntPtrConstant(0)); 7021 } else if (DestVT.bitsGT(MVT::f64)) { 7022 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 7023 } 7024 7025 // Handle final rounding. 7026 return Sub; 7027} 7028 7029SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, 7030 SelectionDAG &DAG) const { 7031 SDValue N0 = Op.getOperand(0); 7032 DebugLoc dl = Op.getDebugLoc(); 7033 7034 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't 7035 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform 7036 // the optimization here. 7037 if (DAG.SignBitIsZero(N0)) 7038 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); 7039 7040 EVT SrcVT = N0.getValueType(); 7041 EVT DstVT = Op.getValueType(); 7042 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) 7043 return LowerUINT_TO_FP_i64(Op, DAG); 7044 else if (SrcVT == MVT::i32 && X86ScalarSSEf64) 7045 return LowerUINT_TO_FP_i32(Op, DAG); 7046 7047 // Make a 64-bit buffer, and use it to build an FILD. 7048 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); 7049 if (SrcVT == MVT::i32) { 7050 SDValue WordOff = DAG.getConstant(4, getPointerTy()); 7051 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, 7052 getPointerTy(), StackSlot, WordOff); 7053 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 7054 StackSlot, MachinePointerInfo(), 7055 false, false, 0); 7056 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), 7057 OffsetSlot, MachinePointerInfo(), 7058 false, false, 0); 7059 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); 7060 return Fild; 7061 } 7062 7063 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); 7064 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 7065 StackSlot, MachinePointerInfo(), 7066 false, false, 0); 7067 // For i64 source, we need to add the appropriate power of 2 if the input 7068 // was negative. This is the same as the optimization in 7069 // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here, 7070 // we must be careful to do the computation in x87 extended precision, not 7071 // in SSE. (The generic code can't know it's OK to do this, or how to.) 7072 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 7073 MachineMemOperand *MMO = 7074 DAG.getMachineFunction() 7075 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7076 MachineMemOperand::MOLoad, 8, 8); 7077 7078 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); 7079 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; 7080 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, 3, 7081 MVT::i64, MMO); 7082 7083 APInt FF(32, 0x5F800000ULL); 7084 7085 // Check whether the sign bit is set. 7086 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 7087 Op.getOperand(0), DAG.getConstant(0, MVT::i64), 7088 ISD::SETLT); 7089 7090 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 7091 SDValue FudgePtr = DAG.getConstantPool( 7092 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 7093 getPointerTy()); 7094 7095 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 7096 SDValue Zero = DAG.getIntPtrConstant(0); 7097 SDValue Four = DAG.getIntPtrConstant(4); 7098 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 7099 Zero, Four); 7100 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset); 7101 7102 // Load the value out, extending it from f32 to f80. 7103 // FIXME: Avoid the extend by constructing the right constant pool? 7104 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), 7105 FudgePtr, MachinePointerInfo::getConstantPool(), 7106 MVT::f32, false, false, 4); 7107 // Extend everything to 80 bits to force it to be done on x87. 7108 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); 7109 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0)); 7110} 7111 7112std::pair<SDValue,SDValue> X86TargetLowering:: 7113FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned) const { 7114 DebugLoc DL = Op.getDebugLoc(); 7115 7116 EVT DstTy = Op.getValueType(); 7117 7118 if (!IsSigned) { 7119 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); 7120 DstTy = MVT::i64; 7121 } 7122 7123 assert(DstTy.getSimpleVT() <= MVT::i64 && 7124 DstTy.getSimpleVT() >= MVT::i16 && 7125 "Unknown FP_TO_SINT to lower!"); 7126 7127 // These are really Legal. 7128 if (DstTy == MVT::i32 && 7129 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 7130 return std::make_pair(SDValue(), SDValue()); 7131 if (Subtarget->is64Bit() && 7132 DstTy == MVT::i64 && 7133 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 7134 return std::make_pair(SDValue(), SDValue()); 7135 7136 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 7137 // stack slot. 7138 MachineFunction &MF = DAG.getMachineFunction(); 7139 unsigned MemSize = DstTy.getSizeInBits()/8; 7140 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false); 7141 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7142 7143 7144 7145 unsigned Opc; 7146 switch (DstTy.getSimpleVT().SimpleTy) { 7147 default: llvm_unreachable("Invalid FP_TO_SINT to lower!"); 7148 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 7149 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 7150 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 7151 } 7152 7153 SDValue Chain = DAG.getEntryNode(); 7154 SDValue Value = Op.getOperand(0); 7155 EVT TheVT = Op.getOperand(0).getValueType(); 7156 if (isScalarFPTypeInSSEReg(TheVT)) { 7157 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 7158 Chain = DAG.getStore(Chain, DL, Value, StackSlot, 7159 MachinePointerInfo::getFixedStack(SSFI), 7160 false, false, 0); 7161 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 7162 SDValue Ops[] = { 7163 Chain, StackSlot, DAG.getValueType(TheVT) 7164 }; 7165 7166 MachineMemOperand *MMO = 7167 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7168 MachineMemOperand::MOLoad, MemSize, MemSize); 7169 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, 3, 7170 DstTy, MMO); 7171 Chain = Value.getValue(1); 7172 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false); 7173 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7174 } 7175 7176 MachineMemOperand *MMO = 7177 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7178 MachineMemOperand::MOStore, MemSize, MemSize); 7179 7180 // Build the FP_TO_INT*_IN_MEM 7181 SDValue Ops[] = { Chain, Value, StackSlot }; 7182 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), 7183 Ops, 3, DstTy, MMO); 7184 7185 return std::make_pair(FIST, StackSlot); 7186} 7187 7188SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, 7189 SelectionDAG &DAG) const { 7190 if (Op.getValueType().isVector()) 7191 return SDValue(); 7192 7193 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, true); 7194 SDValue FIST = Vals.first, StackSlot = Vals.second; 7195 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal. 7196 if (FIST.getNode() == 0) return Op; 7197 7198 // Load the result. 7199 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 7200 FIST, StackSlot, MachinePointerInfo(), false, false, 0); 7201} 7202 7203SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op, 7204 SelectionDAG &DAG) const { 7205 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, false); 7206 SDValue FIST = Vals.first, StackSlot = Vals.second; 7207 assert(FIST.getNode() && "Unexpected failure"); 7208 7209 // Load the result. 7210 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 7211 FIST, StackSlot, MachinePointerInfo(), false, false, 0); 7212} 7213 7214SDValue X86TargetLowering::LowerFABS(SDValue Op, 7215 SelectionDAG &DAG) const { 7216 LLVMContext *Context = DAG.getContext(); 7217 DebugLoc dl = Op.getDebugLoc(); 7218 EVT VT = Op.getValueType(); 7219 EVT EltVT = VT; 7220 if (VT.isVector()) 7221 EltVT = VT.getVectorElementType(); 7222 std::vector<Constant*> CV; 7223 if (EltVT == MVT::f64) { 7224 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63)))); 7225 CV.push_back(C); 7226 CV.push_back(C); 7227 } else { 7228 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31)))); 7229 CV.push_back(C); 7230 CV.push_back(C); 7231 CV.push_back(C); 7232 CV.push_back(C); 7233 } 7234 Constant *C = ConstantVector::get(CV); 7235 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7236 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 7237 MachinePointerInfo::getConstantPool(), 7238 false, false, 16); 7239 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask); 7240} 7241 7242SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { 7243 LLVMContext *Context = DAG.getContext(); 7244 DebugLoc dl = Op.getDebugLoc(); 7245 EVT VT = Op.getValueType(); 7246 EVT EltVT = VT; 7247 if (VT.isVector()) 7248 EltVT = VT.getVectorElementType(); 7249 std::vector<Constant*> CV; 7250 if (EltVT == MVT::f64) { 7251 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63))); 7252 CV.push_back(C); 7253 CV.push_back(C); 7254 } else { 7255 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31))); 7256 CV.push_back(C); 7257 CV.push_back(C); 7258 CV.push_back(C); 7259 CV.push_back(C); 7260 } 7261 Constant *C = ConstantVector::get(CV); 7262 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7263 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 7264 MachinePointerInfo::getConstantPool(), 7265 false, false, 16); 7266 if (VT.isVector()) { 7267 return DAG.getNode(ISD::BITCAST, dl, VT, 7268 DAG.getNode(ISD::XOR, dl, MVT::v2i64, 7269 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 7270 Op.getOperand(0)), 7271 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Mask))); 7272 } else { 7273 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask); 7274 } 7275} 7276 7277SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { 7278 LLVMContext *Context = DAG.getContext(); 7279 SDValue Op0 = Op.getOperand(0); 7280 SDValue Op1 = Op.getOperand(1); 7281 DebugLoc dl = Op.getDebugLoc(); 7282 EVT VT = Op.getValueType(); 7283 EVT SrcVT = Op1.getValueType(); 7284 7285 // If second operand is smaller, extend it first. 7286 if (SrcVT.bitsLT(VT)) { 7287 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1); 7288 SrcVT = VT; 7289 } 7290 // And if it is bigger, shrink it first. 7291 if (SrcVT.bitsGT(VT)) { 7292 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1)); 7293 SrcVT = VT; 7294 } 7295 7296 // At this point the operands and the result should have the same 7297 // type, and that won't be f80 since that is not custom lowered. 7298 7299 // First get the sign bit of second operand. 7300 std::vector<Constant*> CV; 7301 if (SrcVT == MVT::f64) { 7302 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63)))); 7303 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 7304 } else { 7305 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)))); 7306 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7307 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7308 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7309 } 7310 Constant *C = ConstantVector::get(CV); 7311 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7312 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx, 7313 MachinePointerInfo::getConstantPool(), 7314 false, false, 16); 7315 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1); 7316 7317 // Shift sign bit right or left if the two operands have different types. 7318 if (SrcVT.bitsGT(VT)) { 7319 // Op0 is MVT::f32, Op1 is MVT::f64. 7320 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit); 7321 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit, 7322 DAG.getConstant(32, MVT::i32)); 7323 SignBit = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, SignBit); 7324 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit, 7325 DAG.getIntPtrConstant(0)); 7326 } 7327 7328 // Clear first operand sign bit. 7329 CV.clear(); 7330 if (VT == MVT::f64) { 7331 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))))); 7332 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 7333 } else { 7334 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))))); 7335 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7336 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7337 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 7338 } 7339 C = ConstantVector::get(CV); 7340 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7341 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 7342 MachinePointerInfo::getConstantPool(), 7343 false, false, 16); 7344 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2); 7345 7346 // Or the value with the sign bit. 7347 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit); 7348} 7349 7350SDValue X86TargetLowering::LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const { 7351 SDValue N0 = Op.getOperand(0); 7352 DebugLoc dl = Op.getDebugLoc(); 7353 EVT VT = Op.getValueType(); 7354 7355 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1). 7356 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0, 7357 DAG.getConstant(1, VT)); 7358 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT)); 7359} 7360 7361/// Emit nodes that will be selected as "test Op0,Op0", or something 7362/// equivalent. 7363SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, 7364 SelectionDAG &DAG) const { 7365 DebugLoc dl = Op.getDebugLoc(); 7366 7367 // CF and OF aren't always set the way we want. Determine which 7368 // of these we need. 7369 bool NeedCF = false; 7370 bool NeedOF = false; 7371 switch (X86CC) { 7372 default: break; 7373 case X86::COND_A: case X86::COND_AE: 7374 case X86::COND_B: case X86::COND_BE: 7375 NeedCF = true; 7376 break; 7377 case X86::COND_G: case X86::COND_GE: 7378 case X86::COND_L: case X86::COND_LE: 7379 case X86::COND_O: case X86::COND_NO: 7380 NeedOF = true; 7381 break; 7382 } 7383 7384 // See if we can use the EFLAGS value from the operand instead of 7385 // doing a separate TEST. TEST always sets OF and CF to 0, so unless 7386 // we prove that the arithmetic won't overflow, we can't use OF or CF. 7387 if (Op.getResNo() != 0 || NeedOF || NeedCF) 7388 // Emit a CMP with 0, which is the TEST pattern. 7389 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, 7390 DAG.getConstant(0, Op.getValueType())); 7391 7392 unsigned Opcode = 0; 7393 unsigned NumOperands = 0; 7394 switch (Op.getNode()->getOpcode()) { 7395 case ISD::ADD: 7396 // Due to an isel shortcoming, be conservative if this add is likely to be 7397 // selected as part of a load-modify-store instruction. When the root node 7398 // in a match is a store, isel doesn't know how to remap non-chain non-flag 7399 // uses of other nodes in the match, such as the ADD in this case. This 7400 // leads to the ADD being left around and reselected, with the result being 7401 // two adds in the output. Alas, even if none our users are stores, that 7402 // doesn't prove we're O.K. Ergo, if we have any parents that aren't 7403 // CopyToReg or SETCC, eschew INC/DEC. A better fix seems to require 7404 // climbing the DAG back to the root, and it doesn't seem to be worth the 7405 // effort. 7406 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 7407 UE = Op.getNode()->use_end(); UI != UE; ++UI) 7408 if (UI->getOpcode() != ISD::CopyToReg && UI->getOpcode() != ISD::SETCC) 7409 goto default_case; 7410 7411 if (ConstantSDNode *C = 7412 dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) { 7413 // An add of one will be selected as an INC. 7414 if (C->getAPIntValue() == 1) { 7415 Opcode = X86ISD::INC; 7416 NumOperands = 1; 7417 break; 7418 } 7419 7420 // An add of negative one (subtract of one) will be selected as a DEC. 7421 if (C->getAPIntValue().isAllOnesValue()) { 7422 Opcode = X86ISD::DEC; 7423 NumOperands = 1; 7424 break; 7425 } 7426 } 7427 7428 // Otherwise use a regular EFLAGS-setting add. 7429 Opcode = X86ISD::ADD; 7430 NumOperands = 2; 7431 break; 7432 case ISD::AND: { 7433 // If the primary and result isn't used, don't bother using X86ISD::AND, 7434 // because a TEST instruction will be better. 7435 bool NonFlagUse = false; 7436 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 7437 UE = Op.getNode()->use_end(); UI != UE; ++UI) { 7438 SDNode *User = *UI; 7439 unsigned UOpNo = UI.getOperandNo(); 7440 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) { 7441 // Look pass truncate. 7442 UOpNo = User->use_begin().getOperandNo(); 7443 User = *User->use_begin(); 7444 } 7445 7446 if (User->getOpcode() != ISD::BRCOND && 7447 User->getOpcode() != ISD::SETCC && 7448 (User->getOpcode() != ISD::SELECT || UOpNo != 0)) { 7449 NonFlagUse = true; 7450 break; 7451 } 7452 } 7453 7454 if (!NonFlagUse) 7455 break; 7456 } 7457 // FALL THROUGH 7458 case ISD::SUB: 7459 case ISD::OR: 7460 case ISD::XOR: 7461 // Due to the ISEL shortcoming noted above, be conservative if this op is 7462 // likely to be selected as part of a load-modify-store instruction. 7463 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 7464 UE = Op.getNode()->use_end(); UI != UE; ++UI) 7465 if (UI->getOpcode() == ISD::STORE) 7466 goto default_case; 7467 7468 // Otherwise use a regular EFLAGS-setting instruction. 7469 switch (Op.getNode()->getOpcode()) { 7470 default: llvm_unreachable("unexpected operator!"); 7471 case ISD::SUB: Opcode = X86ISD::SUB; break; 7472 case ISD::OR: Opcode = X86ISD::OR; break; 7473 case ISD::XOR: Opcode = X86ISD::XOR; break; 7474 case ISD::AND: Opcode = X86ISD::AND; break; 7475 } 7476 7477 NumOperands = 2; 7478 break; 7479 case X86ISD::ADD: 7480 case X86ISD::SUB: 7481 case X86ISD::INC: 7482 case X86ISD::DEC: 7483 case X86ISD::OR: 7484 case X86ISD::XOR: 7485 case X86ISD::AND: 7486 return SDValue(Op.getNode(), 1); 7487 default: 7488 default_case: 7489 break; 7490 } 7491 7492 if (Opcode == 0) 7493 // Emit a CMP with 0, which is the TEST pattern. 7494 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, 7495 DAG.getConstant(0, Op.getValueType())); 7496 7497 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 7498 SmallVector<SDValue, 4> Ops; 7499 for (unsigned i = 0; i != NumOperands; ++i) 7500 Ops.push_back(Op.getOperand(i)); 7501 7502 SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands); 7503 DAG.ReplaceAllUsesWith(Op, New); 7504 return SDValue(New.getNode(), 1); 7505} 7506 7507/// Emit nodes that will be selected as "cmp Op0,Op1", or something 7508/// equivalent. 7509SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, 7510 SelectionDAG &DAG) const { 7511 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) 7512 if (C->getAPIntValue() == 0) 7513 return EmitTest(Op0, X86CC, DAG); 7514 7515 DebugLoc dl = Op0.getDebugLoc(); 7516 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); 7517} 7518 7519/// LowerToBT - Result of 'and' is compared against zero. Turn it into a BT node 7520/// if it's possible. 7521SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC, 7522 DebugLoc dl, SelectionDAG &DAG) const { 7523 SDValue Op0 = And.getOperand(0); 7524 SDValue Op1 = And.getOperand(1); 7525 if (Op0.getOpcode() == ISD::TRUNCATE) 7526 Op0 = Op0.getOperand(0); 7527 if (Op1.getOpcode() == ISD::TRUNCATE) 7528 Op1 = Op1.getOperand(0); 7529 7530 SDValue LHS, RHS; 7531 if (Op1.getOpcode() == ISD::SHL) 7532 std::swap(Op0, Op1); 7533 if (Op0.getOpcode() == ISD::SHL) { 7534 if (ConstantSDNode *And00C = dyn_cast<ConstantSDNode>(Op0.getOperand(0))) 7535 if (And00C->getZExtValue() == 1) { 7536 // If we looked past a truncate, check that it's only truncating away 7537 // known zeros. 7538 unsigned BitWidth = Op0.getValueSizeInBits(); 7539 unsigned AndBitWidth = And.getValueSizeInBits(); 7540 if (BitWidth > AndBitWidth) { 7541 APInt Mask = APInt::getAllOnesValue(BitWidth), Zeros, Ones; 7542 DAG.ComputeMaskedBits(Op0, Mask, Zeros, Ones); 7543 if (Zeros.countLeadingOnes() < BitWidth - AndBitWidth) 7544 return SDValue(); 7545 } 7546 LHS = Op1; 7547 RHS = Op0.getOperand(1); 7548 } 7549 } else if (Op1.getOpcode() == ISD::Constant) { 7550 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1); 7551 SDValue AndLHS = Op0; 7552 if (AndRHS->getZExtValue() == 1 && AndLHS.getOpcode() == ISD::SRL) { 7553 LHS = AndLHS.getOperand(0); 7554 RHS = AndLHS.getOperand(1); 7555 } 7556 } 7557 7558 if (LHS.getNode()) { 7559 // If LHS is i8, promote it to i32 with any_extend. There is no i8 BT 7560 // instruction. Since the shift amount is in-range-or-undefined, we know 7561 // that doing a bittest on the i32 value is ok. We extend to i32 because 7562 // the encoding for the i16 version is larger than the i32 version. 7563 // Also promote i16 to i32 for performance / code size reason. 7564 if (LHS.getValueType() == MVT::i8 || 7565 LHS.getValueType() == MVT::i16) 7566 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); 7567 7568 // If the operand types disagree, extend the shift amount to match. Since 7569 // BT ignores high bits (like shifts) we can use anyextend. 7570 if (LHS.getValueType() != RHS.getValueType()) 7571 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS); 7572 7573 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); 7574 unsigned Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B; 7575 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 7576 DAG.getConstant(Cond, MVT::i8), BT); 7577 } 7578 7579 return SDValue(); 7580} 7581 7582SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 7583 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 7584 SDValue Op0 = Op.getOperand(0); 7585 SDValue Op1 = Op.getOperand(1); 7586 DebugLoc dl = Op.getDebugLoc(); 7587 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 7588 7589 // Optimize to BT if possible. 7590 // Lower (X & (1 << N)) == 0 to BT(X, N). 7591 // Lower ((X >>u N) & 1) != 0 to BT(X, N). 7592 // Lower ((X >>s N) & 1) != 0 to BT(X, N). 7593 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && 7594 Op1.getOpcode() == ISD::Constant && 7595 cast<ConstantSDNode>(Op1)->isNullValue() && 7596 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 7597 SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG); 7598 if (NewSetCC.getNode()) 7599 return NewSetCC; 7600 } 7601 7602 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms of 7603 // these. 7604 if (Op1.getOpcode() == ISD::Constant && 7605 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 || 7606 cast<ConstantSDNode>(Op1)->isNullValue()) && 7607 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 7608 7609 // If the input is a setcc, then reuse the input setcc or use a new one with 7610 // the inverted condition. 7611 if (Op0.getOpcode() == X86ISD::SETCC) { 7612 X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0); 7613 bool Invert = (CC == ISD::SETNE) ^ 7614 cast<ConstantSDNode>(Op1)->isNullValue(); 7615 if (!Invert) return Op0; 7616 7617 CCode = X86::GetOppositeBranchCondition(CCode); 7618 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 7619 DAG.getConstant(CCode, MVT::i8), Op0.getOperand(1)); 7620 } 7621 } 7622 7623 bool isFP = Op1.getValueType().isFloatingPoint(); 7624 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); 7625 if (X86CC == X86::COND_INVALID) 7626 return SDValue(); 7627 7628 SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, DAG); 7629 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 7630 DAG.getConstant(X86CC, MVT::i8), EFLAGS); 7631} 7632 7633SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const { 7634 SDValue Cond; 7635 SDValue Op0 = Op.getOperand(0); 7636 SDValue Op1 = Op.getOperand(1); 7637 SDValue CC = Op.getOperand(2); 7638 EVT VT = Op.getValueType(); 7639 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 7640 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 7641 DebugLoc dl = Op.getDebugLoc(); 7642 7643 if (isFP) { 7644 unsigned SSECC = 8; 7645 EVT VT0 = Op0.getValueType(); 7646 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); 7647 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; 7648 bool Swap = false; 7649 7650 switch (SetCCOpcode) { 7651 default: break; 7652 case ISD::SETOEQ: 7653 case ISD::SETEQ: SSECC = 0; break; 7654 case ISD::SETOGT: 7655 case ISD::SETGT: Swap = true; // Fallthrough 7656 case ISD::SETLT: 7657 case ISD::SETOLT: SSECC = 1; break; 7658 case ISD::SETOGE: 7659 case ISD::SETGE: Swap = true; // Fallthrough 7660 case ISD::SETLE: 7661 case ISD::SETOLE: SSECC = 2; break; 7662 case ISD::SETUO: SSECC = 3; break; 7663 case ISD::SETUNE: 7664 case ISD::SETNE: SSECC = 4; break; 7665 case ISD::SETULE: Swap = true; 7666 case ISD::SETUGE: SSECC = 5; break; 7667 case ISD::SETULT: Swap = true; 7668 case ISD::SETUGT: SSECC = 6; break; 7669 case ISD::SETO: SSECC = 7; break; 7670 } 7671 if (Swap) 7672 std::swap(Op0, Op1); 7673 7674 // In the two special cases we can't handle, emit two comparisons. 7675 if (SSECC == 8) { 7676 if (SetCCOpcode == ISD::SETUEQ) { 7677 SDValue UNORD, EQ; 7678 UNORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 7679 EQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 7680 return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ); 7681 } 7682 else if (SetCCOpcode == ISD::SETONE) { 7683 SDValue ORD, NEQ; 7684 ORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 7685 NEQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 7686 return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ); 7687 } 7688 llvm_unreachable("Illegal FP comparison"); 7689 } 7690 // Handle all other FP comparisons here. 7691 return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 7692 } 7693 7694 // We are handling one of the integer comparisons here. Since SSE only has 7695 // GT and EQ comparisons for integer, swapping operands and multiple 7696 // operations may be required for some comparisons. 7697 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 7698 bool Swap = false, Invert = false, FlipSigns = false; 7699 7700 switch (VT.getSimpleVT().SimpleTy) { 7701 default: break; 7702 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 7703 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 7704 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 7705 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 7706 } 7707 7708 switch (SetCCOpcode) { 7709 default: break; 7710 case ISD::SETNE: Invert = true; 7711 case ISD::SETEQ: Opc = EQOpc; break; 7712 case ISD::SETLT: Swap = true; 7713 case ISD::SETGT: Opc = GTOpc; break; 7714 case ISD::SETGE: Swap = true; 7715 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 7716 case ISD::SETULT: Swap = true; 7717 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 7718 case ISD::SETUGE: Swap = true; 7719 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 7720 } 7721 if (Swap) 7722 std::swap(Op0, Op1); 7723 7724 // Since SSE has no unsigned integer comparisons, we need to flip the sign 7725 // bits of the inputs before performing those operations. 7726 if (FlipSigns) { 7727 EVT EltVT = VT.getVectorElementType(); 7728 SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), 7729 EltVT); 7730 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 7731 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0], 7732 SignBits.size()); 7733 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec); 7734 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec); 7735 } 7736 7737 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); 7738 7739 // If the logical-not of the result is required, perform that now. 7740 if (Invert) 7741 Result = DAG.getNOT(dl, Result, VT); 7742 7743 return Result; 7744} 7745 7746// isX86LogicalCmp - Return true if opcode is a X86 logical comparison. 7747static bool isX86LogicalCmp(SDValue Op) { 7748 unsigned Opc = Op.getNode()->getOpcode(); 7749 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) 7750 return true; 7751 if (Op.getResNo() == 1 && 7752 (Opc == X86ISD::ADD || 7753 Opc == X86ISD::SUB || 7754 Opc == X86ISD::ADC || 7755 Opc == X86ISD::SBB || 7756 Opc == X86ISD::SMUL || 7757 Opc == X86ISD::UMUL || 7758 Opc == X86ISD::INC || 7759 Opc == X86ISD::DEC || 7760 Opc == X86ISD::OR || 7761 Opc == X86ISD::XOR || 7762 Opc == X86ISD::AND)) 7763 return true; 7764 7765 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL) 7766 return true; 7767 7768 return false; 7769} 7770 7771static bool isZero(SDValue V) { 7772 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V); 7773 return C && C->isNullValue(); 7774} 7775 7776static bool isAllOnes(SDValue V) { 7777 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V); 7778 return C && C->isAllOnesValue(); 7779} 7780 7781SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { 7782 bool addTest = true; 7783 SDValue Cond = Op.getOperand(0); 7784 SDValue Op1 = Op.getOperand(1); 7785 SDValue Op2 = Op.getOperand(2); 7786 DebugLoc DL = Op.getDebugLoc(); 7787 SDValue CC; 7788 7789 if (Cond.getOpcode() == ISD::SETCC) { 7790 SDValue NewCond = LowerSETCC(Cond, DAG); 7791 if (NewCond.getNode()) 7792 Cond = NewCond; 7793 } 7794 7795 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y 7796 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y 7797 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y 7798 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y 7799 if (Cond.getOpcode() == X86ISD::SETCC && 7800 Cond.getOperand(1).getOpcode() == X86ISD::CMP && 7801 isZero(Cond.getOperand(1).getOperand(1))) { 7802 SDValue Cmp = Cond.getOperand(1); 7803 7804 unsigned CondCode =cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue(); 7805 7806 if ((isAllOnes(Op1) || isAllOnes(Op2)) && 7807 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) { 7808 SDValue Y = isAllOnes(Op2) ? Op1 : Op2; 7809 7810 SDValue CmpOp0 = Cmp.getOperand(0); 7811 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, 7812 CmpOp0, DAG.getConstant(1, CmpOp0.getValueType())); 7813 7814 SDValue Res = // Res = 0 or -1. 7815 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), 7816 DAG.getConstant(X86::COND_B, MVT::i8), Cmp); 7817 7818 if (isAllOnes(Op1) != (CondCode == X86::COND_E)) 7819 Res = DAG.getNOT(DL, Res, Res.getValueType()); 7820 7821 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Op2); 7822 if (N2C == 0 || !N2C->isNullValue()) 7823 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y); 7824 return Res; 7825 } 7826 } 7827 7828 // Look past (and (setcc_carry (cmp ...)), 1). 7829 if (Cond.getOpcode() == ISD::AND && 7830 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) { 7831 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 7832 if (C && C->getAPIntValue() == 1) 7833 Cond = Cond.getOperand(0); 7834 } 7835 7836 // If condition flag is set by a X86ISD::CMP, then use it as the condition 7837 // setting operand in place of the X86ISD::SETCC. 7838 if (Cond.getOpcode() == X86ISD::SETCC || 7839 Cond.getOpcode() == X86ISD::SETCC_CARRY) { 7840 CC = Cond.getOperand(0); 7841 7842 SDValue Cmp = Cond.getOperand(1); 7843 unsigned Opc = Cmp.getOpcode(); 7844 EVT VT = Op.getValueType(); 7845 7846 bool IllegalFPCMov = false; 7847 if (VT.isFloatingPoint() && !VT.isVector() && 7848 !isScalarFPTypeInSSEReg(VT)) // FPStack? 7849 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue()); 7850 7851 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) || 7852 Opc == X86ISD::BT) { // FIXME 7853 Cond = Cmp; 7854 addTest = false; 7855 } 7856 } 7857 7858 if (addTest) { 7859 // Look pass the truncate. 7860 if (Cond.getOpcode() == ISD::TRUNCATE) 7861 Cond = Cond.getOperand(0); 7862 7863 // We know the result of AND is compared against zero. Try to match 7864 // it to BT. 7865 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { 7866 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG); 7867 if (NewSetCC.getNode()) { 7868 CC = NewSetCC.getOperand(0); 7869 Cond = NewSetCC.getOperand(1); 7870 addTest = false; 7871 } 7872 } 7873 } 7874 7875 if (addTest) { 7876 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 7877 Cond = EmitTest(Cond, X86::COND_NE, DAG); 7878 } 7879 7880 // a < b ? -1 : 0 -> RES = ~setcc_carry 7881 // a < b ? 0 : -1 -> RES = setcc_carry 7882 // a >= b ? -1 : 0 -> RES = setcc_carry 7883 // a >= b ? 0 : -1 -> RES = ~setcc_carry 7884 if (Cond.getOpcode() == X86ISD::CMP) { 7885 unsigned CondCode = cast<ConstantSDNode>(CC)->getZExtValue(); 7886 7887 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) && 7888 (isAllOnes(Op1) || isAllOnes(Op2)) && (isZero(Op1) || isZero(Op2))) { 7889 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), 7890 DAG.getConstant(X86::COND_B, MVT::i8), Cond); 7891 if (isAllOnes(Op1) != (CondCode == X86::COND_B)) 7892 return DAG.getNOT(DL, Res, Res.getValueType()); 7893 return Res; 7894 } 7895 } 7896 7897 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 7898 // condition is true. 7899 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 7900 SDValue Ops[] = { Op2, Op1, CC, Cond }; 7901 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops, array_lengthof(Ops)); 7902} 7903 7904// isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or 7905// ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart 7906// from the AND / OR. 7907static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) { 7908 Opc = Op.getOpcode(); 7909 if (Opc != ISD::OR && Opc != ISD::AND) 7910 return false; 7911 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC && 7912 Op.getOperand(0).hasOneUse() && 7913 Op.getOperand(1).getOpcode() == X86ISD::SETCC && 7914 Op.getOperand(1).hasOneUse()); 7915} 7916 7917// isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and 7918// 1 and that the SETCC node has a single use. 7919static bool isXor1OfSetCC(SDValue Op) { 7920 if (Op.getOpcode() != ISD::XOR) 7921 return false; 7922 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7923 if (N1C && N1C->getAPIntValue() == 1) { 7924 return Op.getOperand(0).getOpcode() == X86ISD::SETCC && 7925 Op.getOperand(0).hasOneUse(); 7926 } 7927 return false; 7928} 7929 7930SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 7931 bool addTest = true; 7932 SDValue Chain = Op.getOperand(0); 7933 SDValue Cond = Op.getOperand(1); 7934 SDValue Dest = Op.getOperand(2); 7935 DebugLoc dl = Op.getDebugLoc(); 7936 SDValue CC; 7937 7938 if (Cond.getOpcode() == ISD::SETCC) { 7939 SDValue NewCond = LowerSETCC(Cond, DAG); 7940 if (NewCond.getNode()) 7941 Cond = NewCond; 7942 } 7943#if 0 7944 // FIXME: LowerXALUO doesn't handle these!! 7945 else if (Cond.getOpcode() == X86ISD::ADD || 7946 Cond.getOpcode() == X86ISD::SUB || 7947 Cond.getOpcode() == X86ISD::SMUL || 7948 Cond.getOpcode() == X86ISD::UMUL) 7949 Cond = LowerXALUO(Cond, DAG); 7950#endif 7951 7952 // Look pass (and (setcc_carry (cmp ...)), 1). 7953 if (Cond.getOpcode() == ISD::AND && 7954 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) { 7955 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 7956 if (C && C->getAPIntValue() == 1) 7957 Cond = Cond.getOperand(0); 7958 } 7959 7960 // If condition flag is set by a X86ISD::CMP, then use it as the condition 7961 // setting operand in place of the X86ISD::SETCC. 7962 if (Cond.getOpcode() == X86ISD::SETCC || 7963 Cond.getOpcode() == X86ISD::SETCC_CARRY) { 7964 CC = Cond.getOperand(0); 7965 7966 SDValue Cmp = Cond.getOperand(1); 7967 unsigned Opc = Cmp.getOpcode(); 7968 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp?? 7969 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) { 7970 Cond = Cmp; 7971 addTest = false; 7972 } else { 7973 switch (cast<ConstantSDNode>(CC)->getZExtValue()) { 7974 default: break; 7975 case X86::COND_O: 7976 case X86::COND_B: 7977 // These can only come from an arithmetic instruction with overflow, 7978 // e.g. SADDO, UADDO. 7979 Cond = Cond.getNode()->getOperand(1); 7980 addTest = false; 7981 break; 7982 } 7983 } 7984 } else { 7985 unsigned CondOpc; 7986 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) { 7987 SDValue Cmp = Cond.getOperand(0).getOperand(1); 7988 if (CondOpc == ISD::OR) { 7989 // Also, recognize the pattern generated by an FCMP_UNE. We can emit 7990 // two branches instead of an explicit OR instruction with a 7991 // separate test. 7992 if (Cmp == Cond.getOperand(1).getOperand(1) && 7993 isX86LogicalCmp(Cmp)) { 7994 CC = Cond.getOperand(0).getOperand(0); 7995 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 7996 Chain, Dest, CC, Cmp); 7997 CC = Cond.getOperand(1).getOperand(0); 7998 Cond = Cmp; 7999 addTest = false; 8000 } 8001 } else { // ISD::AND 8002 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit 8003 // two branches instead of an explicit AND instruction with a 8004 // separate test. However, we only do this if this block doesn't 8005 // have a fall-through edge, because this requires an explicit 8006 // jmp when the condition is false. 8007 if (Cmp == Cond.getOperand(1).getOperand(1) && 8008 isX86LogicalCmp(Cmp) && 8009 Op.getNode()->hasOneUse()) { 8010 X86::CondCode CCode = 8011 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 8012 CCode = X86::GetOppositeBranchCondition(CCode); 8013 CC = DAG.getConstant(CCode, MVT::i8); 8014 SDNode *User = *Op.getNode()->use_begin(); 8015 // Look for an unconditional branch following this conditional branch. 8016 // We need this because we need to reverse the successors in order 8017 // to implement FCMP_OEQ. 8018 if (User->getOpcode() == ISD::BR) { 8019 SDValue FalseBB = User->getOperand(1); 8020 SDNode *NewBR = 8021 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); 8022 assert(NewBR == User); 8023 (void)NewBR; 8024 Dest = FalseBB; 8025 8026 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 8027 Chain, Dest, CC, Cmp); 8028 X86::CondCode CCode = 8029 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0); 8030 CCode = X86::GetOppositeBranchCondition(CCode); 8031 CC = DAG.getConstant(CCode, MVT::i8); 8032 Cond = Cmp; 8033 addTest = false; 8034 } 8035 } 8036 } 8037 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) { 8038 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition. 8039 // It should be transformed during dag combiner except when the condition 8040 // is set by a arithmetics with overflow node. 8041 X86::CondCode CCode = 8042 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 8043 CCode = X86::GetOppositeBranchCondition(CCode); 8044 CC = DAG.getConstant(CCode, MVT::i8); 8045 Cond = Cond.getOperand(0).getOperand(1); 8046 addTest = false; 8047 } 8048 } 8049 8050 if (addTest) { 8051 // Look pass the truncate. 8052 if (Cond.getOpcode() == ISD::TRUNCATE) 8053 Cond = Cond.getOperand(0); 8054 8055 // We know the result of AND is compared against zero. Try to match 8056 // it to BT. 8057 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { 8058 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG); 8059 if (NewSetCC.getNode()) { 8060 CC = NewSetCC.getOperand(0); 8061 Cond = NewSetCC.getOperand(1); 8062 addTest = false; 8063 } 8064 } 8065 } 8066 8067 if (addTest) { 8068 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 8069 Cond = EmitTest(Cond, X86::COND_NE, DAG); 8070 } 8071 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 8072 Chain, Dest, CC, Cond); 8073} 8074 8075 8076// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 8077// Calls to _alloca is needed to probe the stack when allocating more than 4k 8078// bytes in one go. Touching the stack at 4K increments is necessary to ensure 8079// that the guard pages used by the OS virtual memory manager are allocated in 8080// correct sequence. 8081SDValue 8082X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 8083 SelectionDAG &DAG) const { 8084 assert((Subtarget->isTargetCygMing() || Subtarget->isTargetWindows()) && 8085 "This should be used only on Windows targets"); 8086 assert(!Subtarget->isTargetEnvMacho()); 8087 DebugLoc dl = Op.getDebugLoc(); 8088 8089 // Get the inputs. 8090 SDValue Chain = Op.getOperand(0); 8091 SDValue Size = Op.getOperand(1); 8092 // FIXME: Ensure alignment here 8093 8094 SDValue Flag; 8095 8096 EVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 8097 unsigned Reg = (Subtarget->is64Bit() ? X86::RAX : X86::EAX); 8098 8099 Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag); 8100 Flag = Chain.getValue(1); 8101 8102 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 8103 8104 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag); 8105 Flag = Chain.getValue(1); 8106 8107 Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1); 8108 8109 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 8110 return DAG.getMergeValues(Ops1, 2, dl); 8111} 8112 8113SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { 8114 MachineFunction &MF = DAG.getMachineFunction(); 8115 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 8116 8117 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 8118 DebugLoc DL = Op.getDebugLoc(); 8119 8120 if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) { 8121 // vastart just stores the address of the VarArgsFrameIndex slot into the 8122 // memory location argument. 8123 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 8124 getPointerTy()); 8125 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), 8126 MachinePointerInfo(SV), false, false, 0); 8127 } 8128 8129 // __va_list_tag: 8130 // gp_offset (0 - 6 * 8) 8131 // fp_offset (48 - 48 + 8 * 16) 8132 // overflow_arg_area (point to parameters coming in memory). 8133 // reg_save_area 8134 SmallVector<SDValue, 8> MemOps; 8135 SDValue FIN = Op.getOperand(1); 8136 // Store gp_offset 8137 SDValue Store = DAG.getStore(Op.getOperand(0), DL, 8138 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), 8139 MVT::i32), 8140 FIN, MachinePointerInfo(SV), false, false, 0); 8141 MemOps.push_back(Store); 8142 8143 // Store fp_offset 8144 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 8145 FIN, DAG.getIntPtrConstant(4)); 8146 Store = DAG.getStore(Op.getOperand(0), DL, 8147 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), 8148 MVT::i32), 8149 FIN, MachinePointerInfo(SV, 4), false, false, 0); 8150 MemOps.push_back(Store); 8151 8152 // Store ptr to overflow_arg_area 8153 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 8154 FIN, DAG.getIntPtrConstant(4)); 8155 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 8156 getPointerTy()); 8157 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, 8158 MachinePointerInfo(SV, 8), 8159 false, false, 0); 8160 MemOps.push_back(Store); 8161 8162 // Store ptr to reg_save_area. 8163 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 8164 FIN, DAG.getIntPtrConstant(8)); 8165 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), 8166 getPointerTy()); 8167 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN, 8168 MachinePointerInfo(SV, 16), false, false, 0); 8169 MemOps.push_back(Store); 8170 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 8171 &MemOps[0], MemOps.size()); 8172} 8173 8174SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 8175 assert(Subtarget->is64Bit() && 8176 "LowerVAARG only handles 64-bit va_arg!"); 8177 assert((Subtarget->isTargetLinux() || 8178 Subtarget->isTargetDarwin()) && 8179 "Unhandled target in LowerVAARG"); 8180 assert(Op.getNode()->getNumOperands() == 4); 8181 SDValue Chain = Op.getOperand(0); 8182 SDValue SrcPtr = Op.getOperand(1); 8183 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 8184 unsigned Align = Op.getConstantOperandVal(3); 8185 DebugLoc dl = Op.getDebugLoc(); 8186 8187 EVT ArgVT = Op.getNode()->getValueType(0); 8188 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 8189 uint32_t ArgSize = getTargetData()->getTypeAllocSize(ArgTy); 8190 uint8_t ArgMode; 8191 8192 // Decide which area this value should be read from. 8193 // TODO: Implement the AMD64 ABI in its entirety. This simple 8194 // selection mechanism works only for the basic types. 8195 if (ArgVT == MVT::f80) { 8196 llvm_unreachable("va_arg for f80 not yet implemented"); 8197 } else if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) { 8198 ArgMode = 2; // Argument passed in XMM register. Use fp_offset. 8199 } else if (ArgVT.isInteger() && ArgSize <= 32 /*bytes*/) { 8200 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset. 8201 } else { 8202 llvm_unreachable("Unhandled argument type in LowerVAARG"); 8203 } 8204 8205 if (ArgMode == 2) { 8206 // Sanity Check: Make sure using fp_offset makes sense. 8207 assert(!UseSoftFloat && 8208 !(DAG.getMachineFunction() 8209 .getFunction()->hasFnAttr(Attribute::NoImplicitFloat)) && 8210 Subtarget->hasXMM()); 8211 } 8212 8213 // Insert VAARG_64 node into the DAG 8214 // VAARG_64 returns two values: Variable Argument Address, Chain 8215 SmallVector<SDValue, 11> InstOps; 8216 InstOps.push_back(Chain); 8217 InstOps.push_back(SrcPtr); 8218 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32)); 8219 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8)); 8220 InstOps.push_back(DAG.getConstant(Align, MVT::i32)); 8221 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other); 8222 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl, 8223 VTs, &InstOps[0], InstOps.size(), 8224 MVT::i64, 8225 MachinePointerInfo(SV), 8226 /*Align=*/0, 8227 /*Volatile=*/false, 8228 /*ReadMem=*/true, 8229 /*WriteMem=*/true); 8230 Chain = VAARG.getValue(1); 8231 8232 // Load the next argument and return it 8233 return DAG.getLoad(ArgVT, dl, 8234 Chain, 8235 VAARG, 8236 MachinePointerInfo(), 8237 false, false, 0); 8238} 8239 8240SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { 8241 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 8242 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 8243 SDValue Chain = Op.getOperand(0); 8244 SDValue DstPtr = Op.getOperand(1); 8245 SDValue SrcPtr = Op.getOperand(2); 8246 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 8247 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 8248 DebugLoc DL = Op.getDebugLoc(); 8249 8250 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, 8251 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false, 8252 false, 8253 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 8254} 8255 8256SDValue 8257X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { 8258 DebugLoc dl = Op.getDebugLoc(); 8259 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 8260 switch (IntNo) { 8261 default: return SDValue(); // Don't custom lower most intrinsics. 8262 // Comparison intrinsics. 8263 case Intrinsic::x86_sse_comieq_ss: 8264 case Intrinsic::x86_sse_comilt_ss: 8265 case Intrinsic::x86_sse_comile_ss: 8266 case Intrinsic::x86_sse_comigt_ss: 8267 case Intrinsic::x86_sse_comige_ss: 8268 case Intrinsic::x86_sse_comineq_ss: 8269 case Intrinsic::x86_sse_ucomieq_ss: 8270 case Intrinsic::x86_sse_ucomilt_ss: 8271 case Intrinsic::x86_sse_ucomile_ss: 8272 case Intrinsic::x86_sse_ucomigt_ss: 8273 case Intrinsic::x86_sse_ucomige_ss: 8274 case Intrinsic::x86_sse_ucomineq_ss: 8275 case Intrinsic::x86_sse2_comieq_sd: 8276 case Intrinsic::x86_sse2_comilt_sd: 8277 case Intrinsic::x86_sse2_comile_sd: 8278 case Intrinsic::x86_sse2_comigt_sd: 8279 case Intrinsic::x86_sse2_comige_sd: 8280 case Intrinsic::x86_sse2_comineq_sd: 8281 case Intrinsic::x86_sse2_ucomieq_sd: 8282 case Intrinsic::x86_sse2_ucomilt_sd: 8283 case Intrinsic::x86_sse2_ucomile_sd: 8284 case Intrinsic::x86_sse2_ucomigt_sd: 8285 case Intrinsic::x86_sse2_ucomige_sd: 8286 case Intrinsic::x86_sse2_ucomineq_sd: { 8287 unsigned Opc = 0; 8288 ISD::CondCode CC = ISD::SETCC_INVALID; 8289 switch (IntNo) { 8290 default: break; 8291 case Intrinsic::x86_sse_comieq_ss: 8292 case Intrinsic::x86_sse2_comieq_sd: 8293 Opc = X86ISD::COMI; 8294 CC = ISD::SETEQ; 8295 break; 8296 case Intrinsic::x86_sse_comilt_ss: 8297 case Intrinsic::x86_sse2_comilt_sd: 8298 Opc = X86ISD::COMI; 8299 CC = ISD::SETLT; 8300 break; 8301 case Intrinsic::x86_sse_comile_ss: 8302 case Intrinsic::x86_sse2_comile_sd: 8303 Opc = X86ISD::COMI; 8304 CC = ISD::SETLE; 8305 break; 8306 case Intrinsic::x86_sse_comigt_ss: 8307 case Intrinsic::x86_sse2_comigt_sd: 8308 Opc = X86ISD::COMI; 8309 CC = ISD::SETGT; 8310 break; 8311 case Intrinsic::x86_sse_comige_ss: 8312 case Intrinsic::x86_sse2_comige_sd: 8313 Opc = X86ISD::COMI; 8314 CC = ISD::SETGE; 8315 break; 8316 case Intrinsic::x86_sse_comineq_ss: 8317 case Intrinsic::x86_sse2_comineq_sd: 8318 Opc = X86ISD::COMI; 8319 CC = ISD::SETNE; 8320 break; 8321 case Intrinsic::x86_sse_ucomieq_ss: 8322 case Intrinsic::x86_sse2_ucomieq_sd: 8323 Opc = X86ISD::UCOMI; 8324 CC = ISD::SETEQ; 8325 break; 8326 case Intrinsic::x86_sse_ucomilt_ss: 8327 case Intrinsic::x86_sse2_ucomilt_sd: 8328 Opc = X86ISD::UCOMI; 8329 CC = ISD::SETLT; 8330 break; 8331 case Intrinsic::x86_sse_ucomile_ss: 8332 case Intrinsic::x86_sse2_ucomile_sd: 8333 Opc = X86ISD::UCOMI; 8334 CC = ISD::SETLE; 8335 break; 8336 case Intrinsic::x86_sse_ucomigt_ss: 8337 case Intrinsic::x86_sse2_ucomigt_sd: 8338 Opc = X86ISD::UCOMI; 8339 CC = ISD::SETGT; 8340 break; 8341 case Intrinsic::x86_sse_ucomige_ss: 8342 case Intrinsic::x86_sse2_ucomige_sd: 8343 Opc = X86ISD::UCOMI; 8344 CC = ISD::SETGE; 8345 break; 8346 case Intrinsic::x86_sse_ucomineq_ss: 8347 case Intrinsic::x86_sse2_ucomineq_sd: 8348 Opc = X86ISD::UCOMI; 8349 CC = ISD::SETNE; 8350 break; 8351 } 8352 8353 SDValue LHS = Op.getOperand(1); 8354 SDValue RHS = Op.getOperand(2); 8355 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG); 8356 assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!"); 8357 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS); 8358 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 8359 DAG.getConstant(X86CC, MVT::i8), Cond); 8360 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 8361 } 8362 // ptest and testp intrinsics. The intrinsic these come from are designed to 8363 // return an integer value, not just an instruction so lower it to the ptest 8364 // or testp pattern and a setcc for the result. 8365 case Intrinsic::x86_sse41_ptestz: 8366 case Intrinsic::x86_sse41_ptestc: 8367 case Intrinsic::x86_sse41_ptestnzc: 8368 case Intrinsic::x86_avx_ptestz_256: 8369 case Intrinsic::x86_avx_ptestc_256: 8370 case Intrinsic::x86_avx_ptestnzc_256: 8371 case Intrinsic::x86_avx_vtestz_ps: 8372 case Intrinsic::x86_avx_vtestc_ps: 8373 case Intrinsic::x86_avx_vtestnzc_ps: 8374 case Intrinsic::x86_avx_vtestz_pd: 8375 case Intrinsic::x86_avx_vtestc_pd: 8376 case Intrinsic::x86_avx_vtestnzc_pd: 8377 case Intrinsic::x86_avx_vtestz_ps_256: 8378 case Intrinsic::x86_avx_vtestc_ps_256: 8379 case Intrinsic::x86_avx_vtestnzc_ps_256: 8380 case Intrinsic::x86_avx_vtestz_pd_256: 8381 case Intrinsic::x86_avx_vtestc_pd_256: 8382 case Intrinsic::x86_avx_vtestnzc_pd_256: { 8383 bool IsTestPacked = false; 8384 unsigned X86CC = 0; 8385 switch (IntNo) { 8386 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering."); 8387 case Intrinsic::x86_avx_vtestz_ps: 8388 case Intrinsic::x86_avx_vtestz_pd: 8389 case Intrinsic::x86_avx_vtestz_ps_256: 8390 case Intrinsic::x86_avx_vtestz_pd_256: 8391 IsTestPacked = true; // Fallthrough 8392 case Intrinsic::x86_sse41_ptestz: 8393 case Intrinsic::x86_avx_ptestz_256: 8394 // ZF = 1 8395 X86CC = X86::COND_E; 8396 break; 8397 case Intrinsic::x86_avx_vtestc_ps: 8398 case Intrinsic::x86_avx_vtestc_pd: 8399 case Intrinsic::x86_avx_vtestc_ps_256: 8400 case Intrinsic::x86_avx_vtestc_pd_256: 8401 IsTestPacked = true; // Fallthrough 8402 case Intrinsic::x86_sse41_ptestc: 8403 case Intrinsic::x86_avx_ptestc_256: 8404 // CF = 1 8405 X86CC = X86::COND_B; 8406 break; 8407 case Intrinsic::x86_avx_vtestnzc_ps: 8408 case Intrinsic::x86_avx_vtestnzc_pd: 8409 case Intrinsic::x86_avx_vtestnzc_ps_256: 8410 case Intrinsic::x86_avx_vtestnzc_pd_256: 8411 IsTestPacked = true; // Fallthrough 8412 case Intrinsic::x86_sse41_ptestnzc: 8413 case Intrinsic::x86_avx_ptestnzc_256: 8414 // ZF and CF = 0 8415 X86CC = X86::COND_A; 8416 break; 8417 } 8418 8419 SDValue LHS = Op.getOperand(1); 8420 SDValue RHS = Op.getOperand(2); 8421 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST; 8422 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); 8423 SDValue CC = DAG.getConstant(X86CC, MVT::i8); 8424 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); 8425 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 8426 } 8427 8428 // Fix vector shift instructions where the last operand is a non-immediate 8429 // i32 value. 8430 case Intrinsic::x86_sse2_pslli_w: 8431 case Intrinsic::x86_sse2_pslli_d: 8432 case Intrinsic::x86_sse2_pslli_q: 8433 case Intrinsic::x86_sse2_psrli_w: 8434 case Intrinsic::x86_sse2_psrli_d: 8435 case Intrinsic::x86_sse2_psrli_q: 8436 case Intrinsic::x86_sse2_psrai_w: 8437 case Intrinsic::x86_sse2_psrai_d: 8438 case Intrinsic::x86_mmx_pslli_w: 8439 case Intrinsic::x86_mmx_pslli_d: 8440 case Intrinsic::x86_mmx_pslli_q: 8441 case Intrinsic::x86_mmx_psrli_w: 8442 case Intrinsic::x86_mmx_psrli_d: 8443 case Intrinsic::x86_mmx_psrli_q: 8444 case Intrinsic::x86_mmx_psrai_w: 8445 case Intrinsic::x86_mmx_psrai_d: { 8446 SDValue ShAmt = Op.getOperand(2); 8447 if (isa<ConstantSDNode>(ShAmt)) 8448 return SDValue(); 8449 8450 unsigned NewIntNo = 0; 8451 EVT ShAmtVT = MVT::v4i32; 8452 switch (IntNo) { 8453 case Intrinsic::x86_sse2_pslli_w: 8454 NewIntNo = Intrinsic::x86_sse2_psll_w; 8455 break; 8456 case Intrinsic::x86_sse2_pslli_d: 8457 NewIntNo = Intrinsic::x86_sse2_psll_d; 8458 break; 8459 case Intrinsic::x86_sse2_pslli_q: 8460 NewIntNo = Intrinsic::x86_sse2_psll_q; 8461 break; 8462 case Intrinsic::x86_sse2_psrli_w: 8463 NewIntNo = Intrinsic::x86_sse2_psrl_w; 8464 break; 8465 case Intrinsic::x86_sse2_psrli_d: 8466 NewIntNo = Intrinsic::x86_sse2_psrl_d; 8467 break; 8468 case Intrinsic::x86_sse2_psrli_q: 8469 NewIntNo = Intrinsic::x86_sse2_psrl_q; 8470 break; 8471 case Intrinsic::x86_sse2_psrai_w: 8472 NewIntNo = Intrinsic::x86_sse2_psra_w; 8473 break; 8474 case Intrinsic::x86_sse2_psrai_d: 8475 NewIntNo = Intrinsic::x86_sse2_psra_d; 8476 break; 8477 default: { 8478 ShAmtVT = MVT::v2i32; 8479 switch (IntNo) { 8480 case Intrinsic::x86_mmx_pslli_w: 8481 NewIntNo = Intrinsic::x86_mmx_psll_w; 8482 break; 8483 case Intrinsic::x86_mmx_pslli_d: 8484 NewIntNo = Intrinsic::x86_mmx_psll_d; 8485 break; 8486 case Intrinsic::x86_mmx_pslli_q: 8487 NewIntNo = Intrinsic::x86_mmx_psll_q; 8488 break; 8489 case Intrinsic::x86_mmx_psrli_w: 8490 NewIntNo = Intrinsic::x86_mmx_psrl_w; 8491 break; 8492 case Intrinsic::x86_mmx_psrli_d: 8493 NewIntNo = Intrinsic::x86_mmx_psrl_d; 8494 break; 8495 case Intrinsic::x86_mmx_psrli_q: 8496 NewIntNo = Intrinsic::x86_mmx_psrl_q; 8497 break; 8498 case Intrinsic::x86_mmx_psrai_w: 8499 NewIntNo = Intrinsic::x86_mmx_psra_w; 8500 break; 8501 case Intrinsic::x86_mmx_psrai_d: 8502 NewIntNo = Intrinsic::x86_mmx_psra_d; 8503 break; 8504 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 8505 } 8506 break; 8507 } 8508 } 8509 8510 // The vector shift intrinsics with scalars uses 32b shift amounts but 8511 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits 8512 // to be zero. 8513 SDValue ShOps[4]; 8514 ShOps[0] = ShAmt; 8515 ShOps[1] = DAG.getConstant(0, MVT::i32); 8516 if (ShAmtVT == MVT::v4i32) { 8517 ShOps[2] = DAG.getUNDEF(MVT::i32); 8518 ShOps[3] = DAG.getUNDEF(MVT::i32); 8519 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 4); 8520 } else { 8521 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 2); 8522// FIXME this must be lowered to get rid of the invalid type. 8523 } 8524 8525 EVT VT = Op.getValueType(); 8526 ShAmt = DAG.getNode(ISD::BITCAST, dl, VT, ShAmt); 8527 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8528 DAG.getConstant(NewIntNo, MVT::i32), 8529 Op.getOperand(1), ShAmt); 8530 } 8531 } 8532} 8533 8534SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, 8535 SelectionDAG &DAG) const { 8536 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 8537 MFI->setReturnAddressIsTaken(true); 8538 8539 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 8540 DebugLoc dl = Op.getDebugLoc(); 8541 8542 if (Depth > 0) { 8543 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 8544 SDValue Offset = 8545 DAG.getConstant(TD->getPointerSize(), 8546 Subtarget->is64Bit() ? MVT::i64 : MVT::i32); 8547 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 8548 DAG.getNode(ISD::ADD, dl, getPointerTy(), 8549 FrameAddr, Offset), 8550 MachinePointerInfo(), false, false, 0); 8551 } 8552 8553 // Just load the return address. 8554 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 8555 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 8556 RetAddrFI, MachinePointerInfo(), false, false, 0); 8557} 8558 8559SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { 8560 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 8561 MFI->setFrameAddressIsTaken(true); 8562 8563 EVT VT = Op.getValueType(); 8564 DebugLoc dl = Op.getDebugLoc(); // FIXME probably not meaningful 8565 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 8566 unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP; 8567 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); 8568 while (Depth--) 8569 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, 8570 MachinePointerInfo(), 8571 false, false, 0); 8572 return FrameAddr; 8573} 8574 8575SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 8576 SelectionDAG &DAG) const { 8577 return DAG.getIntPtrConstant(2*TD->getPointerSize()); 8578} 8579 8580SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { 8581 MachineFunction &MF = DAG.getMachineFunction(); 8582 SDValue Chain = Op.getOperand(0); 8583 SDValue Offset = Op.getOperand(1); 8584 SDValue Handler = Op.getOperand(2); 8585 DebugLoc dl = Op.getDebugLoc(); 8586 8587 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, 8588 Subtarget->is64Bit() ? X86::RBP : X86::EBP, 8589 getPointerTy()); 8590 unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX); 8591 8592 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Frame, 8593 DAG.getIntPtrConstant(TD->getPointerSize())); 8594 StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset); 8595 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(), 8596 false, false, 0); 8597 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr); 8598 MF.getRegInfo().addLiveOut(StoreAddrReg); 8599 8600 return DAG.getNode(X86ISD::EH_RETURN, dl, 8601 MVT::Other, 8602 Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); 8603} 8604 8605SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, 8606 SelectionDAG &DAG) const { 8607 SDValue Root = Op.getOperand(0); 8608 SDValue Trmp = Op.getOperand(1); // trampoline 8609 SDValue FPtr = Op.getOperand(2); // nested function 8610 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 8611 DebugLoc dl = Op.getDebugLoc(); 8612 8613 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 8614 8615 if (Subtarget->is64Bit()) { 8616 SDValue OutChains[6]; 8617 8618 // Large code-model. 8619 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode. 8620 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode. 8621 8622 const unsigned char N86R10 = X86_MC::getX86RegNum(X86::R10); 8623 const unsigned char N86R11 = X86_MC::getX86RegNum(X86::R11); 8624 8625 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 8626 8627 // Load the pointer to the nested function into R11. 8628 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 8629 SDValue Addr = Trmp; 8630 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 8631 Addr, MachinePointerInfo(TrmpAddr), 8632 false, false, 0); 8633 8634 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 8635 DAG.getConstant(2, MVT::i64)); 8636 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, 8637 MachinePointerInfo(TrmpAddr, 2), 8638 false, false, 2); 8639 8640 // Load the 'nest' parameter value into R10. 8641 // R10 is specified in X86CallingConv.td 8642 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 8643 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 8644 DAG.getConstant(10, MVT::i64)); 8645 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 8646 Addr, MachinePointerInfo(TrmpAddr, 10), 8647 false, false, 0); 8648 8649 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 8650 DAG.getConstant(12, MVT::i64)); 8651 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, 8652 MachinePointerInfo(TrmpAddr, 12), 8653 false, false, 2); 8654 8655 // Jump to the nested function. 8656 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 8657 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 8658 DAG.getConstant(20, MVT::i64)); 8659 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 8660 Addr, MachinePointerInfo(TrmpAddr, 20), 8661 false, false, 0); 8662 8663 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 8664 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 8665 DAG.getConstant(22, MVT::i64)); 8666 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, 8667 MachinePointerInfo(TrmpAddr, 22), 8668 false, false, 0); 8669 8670 SDValue Ops[] = 8671 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) }; 8672 return DAG.getMergeValues(Ops, 2, dl); 8673 } else { 8674 const Function *Func = 8675 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 8676 CallingConv::ID CC = Func->getCallingConv(); 8677 unsigned NestReg; 8678 8679 switch (CC) { 8680 default: 8681 llvm_unreachable("Unsupported calling convention"); 8682 case CallingConv::C: 8683 case CallingConv::X86_StdCall: { 8684 // Pass 'nest' parameter in ECX. 8685 // Must be kept in sync with X86CallingConv.td 8686 NestReg = X86::ECX; 8687 8688 // Check that ECX wasn't needed by an 'inreg' parameter. 8689 FunctionType *FTy = Func->getFunctionType(); 8690 const AttrListPtr &Attrs = Func->getAttributes(); 8691 8692 if (!Attrs.isEmpty() && !Func->isVarArg()) { 8693 unsigned InRegCount = 0; 8694 unsigned Idx = 1; 8695 8696 for (FunctionType::param_iterator I = FTy->param_begin(), 8697 E = FTy->param_end(); I != E; ++I, ++Idx) 8698 if (Attrs.paramHasAttr(Idx, Attribute::InReg)) 8699 // FIXME: should only count parameters that are lowered to integers. 8700 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32; 8701 8702 if (InRegCount > 2) { 8703 report_fatal_error("Nest register in use - reduce number of inreg" 8704 " parameters!"); 8705 } 8706 } 8707 break; 8708 } 8709 case CallingConv::X86_FastCall: 8710 case CallingConv::X86_ThisCall: 8711 case CallingConv::Fast: 8712 // Pass 'nest' parameter in EAX. 8713 // Must be kept in sync with X86CallingConv.td 8714 NestReg = X86::EAX; 8715 break; 8716 } 8717 8718 SDValue OutChains[4]; 8719 SDValue Addr, Disp; 8720 8721 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 8722 DAG.getConstant(10, MVT::i32)); 8723 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); 8724 8725 // This is storing the opcode for MOV32ri. 8726 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte. 8727 const unsigned char N86Reg = X86_MC::getX86RegNum(NestReg); 8728 OutChains[0] = DAG.getStore(Root, dl, 8729 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 8730 Trmp, MachinePointerInfo(TrmpAddr), 8731 false, false, 0); 8732 8733 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 8734 DAG.getConstant(1, MVT::i32)); 8735 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, 8736 MachinePointerInfo(TrmpAddr, 1), 8737 false, false, 1); 8738 8739 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode. 8740 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 8741 DAG.getConstant(5, MVT::i32)); 8742 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, 8743 MachinePointerInfo(TrmpAddr, 5), 8744 false, false, 1); 8745 8746 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 8747 DAG.getConstant(6, MVT::i32)); 8748 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, 8749 MachinePointerInfo(TrmpAddr, 6), 8750 false, false, 1); 8751 8752 SDValue Ops[] = 8753 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) }; 8754 return DAG.getMergeValues(Ops, 2, dl); 8755 } 8756} 8757 8758SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, 8759 SelectionDAG &DAG) const { 8760 /* 8761 The rounding mode is in bits 11:10 of FPSR, and has the following 8762 settings: 8763 00 Round to nearest 8764 01 Round to -inf 8765 10 Round to +inf 8766 11 Round to 0 8767 8768 FLT_ROUNDS, on the other hand, expects the following: 8769 -1 Undefined 8770 0 Round to 0 8771 1 Round to nearest 8772 2 Round to +inf 8773 3 Round to -inf 8774 8775 To perform the conversion, we do: 8776 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 8777 */ 8778 8779 MachineFunction &MF = DAG.getMachineFunction(); 8780 const TargetMachine &TM = MF.getTarget(); 8781 const TargetFrameLowering &TFI = *TM.getFrameLowering(); 8782 unsigned StackAlignment = TFI.getStackAlignment(); 8783 EVT VT = Op.getValueType(); 8784 DebugLoc DL = Op.getDebugLoc(); 8785 8786 // Save FP Control Word to stack slot 8787 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false); 8788 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 8789 8790 8791 MachineMemOperand *MMO = 8792 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 8793 MachineMemOperand::MOStore, 2, 2); 8794 8795 SDValue Ops[] = { DAG.getEntryNode(), StackSlot }; 8796 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL, 8797 DAG.getVTList(MVT::Other), 8798 Ops, 2, MVT::i16, MMO); 8799 8800 // Load FP Control Word from stack slot 8801 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, 8802 MachinePointerInfo(), false, false, 0); 8803 8804 // Transform as necessary 8805 SDValue CWD1 = 8806 DAG.getNode(ISD::SRL, DL, MVT::i16, 8807 DAG.getNode(ISD::AND, DL, MVT::i16, 8808 CWD, DAG.getConstant(0x800, MVT::i16)), 8809 DAG.getConstant(11, MVT::i8)); 8810 SDValue CWD2 = 8811 DAG.getNode(ISD::SRL, DL, MVT::i16, 8812 DAG.getNode(ISD::AND, DL, MVT::i16, 8813 CWD, DAG.getConstant(0x400, MVT::i16)), 8814 DAG.getConstant(9, MVT::i8)); 8815 8816 SDValue RetVal = 8817 DAG.getNode(ISD::AND, DL, MVT::i16, 8818 DAG.getNode(ISD::ADD, DL, MVT::i16, 8819 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), 8820 DAG.getConstant(1, MVT::i16)), 8821 DAG.getConstant(3, MVT::i16)); 8822 8823 8824 return DAG.getNode((VT.getSizeInBits() < 16 ? 8825 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal); 8826} 8827 8828SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const { 8829 EVT VT = Op.getValueType(); 8830 EVT OpVT = VT; 8831 unsigned NumBits = VT.getSizeInBits(); 8832 DebugLoc dl = Op.getDebugLoc(); 8833 8834 Op = Op.getOperand(0); 8835 if (VT == MVT::i8) { 8836 // Zero extend to i32 since there is not an i8 bsr. 8837 OpVT = MVT::i32; 8838 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 8839 } 8840 8841 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 8842 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 8843 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); 8844 8845 // If src is zero (i.e. bsr sets ZF), returns NumBits. 8846 SDValue Ops[] = { 8847 Op, 8848 DAG.getConstant(NumBits+NumBits-1, OpVT), 8849 DAG.getConstant(X86::COND_E, MVT::i8), 8850 Op.getValue(1) 8851 }; 8852 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops)); 8853 8854 // Finally xor with NumBits-1. 8855 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 8856 8857 if (VT == MVT::i8) 8858 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 8859 return Op; 8860} 8861 8862SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const { 8863 EVT VT = Op.getValueType(); 8864 EVT OpVT = VT; 8865 unsigned NumBits = VT.getSizeInBits(); 8866 DebugLoc dl = Op.getDebugLoc(); 8867 8868 Op = Op.getOperand(0); 8869 if (VT == MVT::i8) { 8870 OpVT = MVT::i32; 8871 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 8872 } 8873 8874 // Issue a bsf (scan bits forward) which also sets EFLAGS. 8875 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 8876 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op); 8877 8878 // If src is zero (i.e. bsf sets ZF), returns NumBits. 8879 SDValue Ops[] = { 8880 Op, 8881 DAG.getConstant(NumBits, OpVT), 8882 DAG.getConstant(X86::COND_E, MVT::i8), 8883 Op.getValue(1) 8884 }; 8885 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops)); 8886 8887 if (VT == MVT::i8) 8888 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 8889 return Op; 8890} 8891 8892SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const { 8893 EVT VT = Op.getValueType(); 8894 assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply"); 8895 DebugLoc dl = Op.getDebugLoc(); 8896 8897 // ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32); 8898 // ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32); 8899 // ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b ); 8900 // ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi ); 8901 // ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b ); 8902 // 8903 // AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 ); 8904 // AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 ); 8905 // return AloBlo + AloBhi + AhiBlo; 8906 8907 SDValue A = Op.getOperand(0); 8908 SDValue B = Op.getOperand(1); 8909 8910 SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8911 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 8912 A, DAG.getConstant(32, MVT::i32)); 8913 SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8914 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 8915 B, DAG.getConstant(32, MVT::i32)); 8916 SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8917 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 8918 A, B); 8919 SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8920 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 8921 A, Bhi); 8922 SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8923 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 8924 Ahi, B); 8925 AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8926 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 8927 AloBhi, DAG.getConstant(32, MVT::i32)); 8928 AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8929 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 8930 AhiBlo, DAG.getConstant(32, MVT::i32)); 8931 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi); 8932 Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo); 8933 return Res; 8934} 8935 8936SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { 8937 8938 EVT VT = Op.getValueType(); 8939 DebugLoc dl = Op.getDebugLoc(); 8940 SDValue R = Op.getOperand(0); 8941 SDValue Amt = Op.getOperand(1); 8942 8943 LLVMContext *Context = DAG.getContext(); 8944 8945 // Must have SSE2. 8946 if (!Subtarget->hasSSE2()) return SDValue(); 8947 8948 // Optimize shl/srl/sra with constant shift amount. 8949 if (isSplatVector(Amt.getNode())) { 8950 SDValue SclrAmt = Amt->getOperand(0); 8951 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SclrAmt)) { 8952 uint64_t ShiftAmt = C->getZExtValue(); 8953 8954 if (VT == MVT::v2i64 && Op.getOpcode() == ISD::SHL) 8955 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8956 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 8957 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8958 8959 if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SHL) 8960 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8961 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), 8962 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8963 8964 if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SHL) 8965 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8966 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), 8967 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8968 8969 if (VT == MVT::v2i64 && Op.getOpcode() == ISD::SRL) 8970 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8971 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 8972 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8973 8974 if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SRL) 8975 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8976 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32), 8977 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8978 8979 if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SRL) 8980 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8981 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32), 8982 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8983 8984 if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SRA) 8985 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8986 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32), 8987 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8988 8989 if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SRA) 8990 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 8991 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32), 8992 R, DAG.getConstant(ShiftAmt, MVT::i32)); 8993 } 8994 } 8995 8996 // Lower SHL with variable shift amount. 8997 // Cannot lower SHL without SSE2 or later. 8998 if (!Subtarget->hasSSE2()) return SDValue(); 8999 9000 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { 9001 Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9002 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), 9003 Op.getOperand(1), DAG.getConstant(23, MVT::i32)); 9004 9005 ConstantInt *CI = ConstantInt::get(*Context, APInt(32, 0x3f800000U)); 9006 9007 std::vector<Constant*> CV(4, CI); 9008 Constant *C = ConstantVector::get(CV); 9009 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 9010 SDValue Addend = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 9011 MachinePointerInfo::getConstantPool(), 9012 false, false, 16); 9013 9014 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Addend); 9015 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); 9016 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); 9017 return DAG.getNode(ISD::MUL, dl, VT, Op, R); 9018 } 9019 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { 9020 // a = a << 5; 9021 Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9022 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), 9023 Op.getOperand(1), DAG.getConstant(5, MVT::i32)); 9024 9025 ConstantInt *CM1 = ConstantInt::get(*Context, APInt(8, 15)); 9026 ConstantInt *CM2 = ConstantInt::get(*Context, APInt(8, 63)); 9027 9028 std::vector<Constant*> CVM1(16, CM1); 9029 std::vector<Constant*> CVM2(16, CM2); 9030 Constant *C = ConstantVector::get(CVM1); 9031 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 9032 SDValue M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 9033 MachinePointerInfo::getConstantPool(), 9034 false, false, 16); 9035 9036 // r = pblendv(r, psllw(r & (char16)15, 4), a); 9037 M = DAG.getNode(ISD::AND, dl, VT, R, M); 9038 M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9039 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M, 9040 DAG.getConstant(4, MVT::i32)); 9041 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op); 9042 // a += a 9043 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); 9044 9045 C = ConstantVector::get(CVM2); 9046 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 9047 M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 9048 MachinePointerInfo::getConstantPool(), 9049 false, false, 16); 9050 9051 // r = pblendv(r, psllw(r & (char16)63, 2), a); 9052 M = DAG.getNode(ISD::AND, dl, VT, R, M); 9053 M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9054 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M, 9055 DAG.getConstant(2, MVT::i32)); 9056 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op); 9057 // a += a 9058 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); 9059 9060 // return pblendv(r, r+r, a); 9061 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, 9062 R, DAG.getNode(ISD::ADD, dl, VT, R, R), Op); 9063 return R; 9064 } 9065 return SDValue(); 9066} 9067 9068SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const { 9069 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus 9070 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering 9071 // looks for this combo and may remove the "setcc" instruction if the "setcc" 9072 // has only one use. 9073 SDNode *N = Op.getNode(); 9074 SDValue LHS = N->getOperand(0); 9075 SDValue RHS = N->getOperand(1); 9076 unsigned BaseOp = 0; 9077 unsigned Cond = 0; 9078 DebugLoc DL = Op.getDebugLoc(); 9079 switch (Op.getOpcode()) { 9080 default: llvm_unreachable("Unknown ovf instruction!"); 9081 case ISD::SADDO: 9082 // A subtract of one will be selected as a INC. Note that INC doesn't 9083 // set CF, so we can't do this for UADDO. 9084 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) 9085 if (C->isOne()) { 9086 BaseOp = X86ISD::INC; 9087 Cond = X86::COND_O; 9088 break; 9089 } 9090 BaseOp = X86ISD::ADD; 9091 Cond = X86::COND_O; 9092 break; 9093 case ISD::UADDO: 9094 BaseOp = X86ISD::ADD; 9095 Cond = X86::COND_B; 9096 break; 9097 case ISD::SSUBO: 9098 // A subtract of one will be selected as a DEC. Note that DEC doesn't 9099 // set CF, so we can't do this for USUBO. 9100 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) 9101 if (C->isOne()) { 9102 BaseOp = X86ISD::DEC; 9103 Cond = X86::COND_O; 9104 break; 9105 } 9106 BaseOp = X86ISD::SUB; 9107 Cond = X86::COND_O; 9108 break; 9109 case ISD::USUBO: 9110 BaseOp = X86ISD::SUB; 9111 Cond = X86::COND_B; 9112 break; 9113 case ISD::SMULO: 9114 BaseOp = X86ISD::SMUL; 9115 Cond = X86::COND_O; 9116 break; 9117 case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs 9118 SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0), 9119 MVT::i32); 9120 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS); 9121 9122 SDValue SetCC = 9123 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 9124 DAG.getConstant(X86::COND_O, MVT::i32), 9125 SDValue(Sum.getNode(), 2)); 9126 9127 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); 9128 return Sum; 9129 } 9130 } 9131 9132 // Also sets EFLAGS. 9133 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); 9134 SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); 9135 9136 SDValue SetCC = 9137 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1), 9138 DAG.getConstant(Cond, MVT::i32), 9139 SDValue(Sum.getNode(), 1)); 9140 9141 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); 9142 return Sum; 9143} 9144 9145SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const{ 9146 DebugLoc dl = Op.getDebugLoc(); 9147 SDNode* Node = Op.getNode(); 9148 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 9149 EVT VT = Node->getValueType(0); 9150 9151 if (Subtarget->hasSSE2() && VT.isVector()) { 9152 unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 9153 ExtraVT.getScalarType().getSizeInBits(); 9154 SDValue ShAmt = DAG.getConstant(BitsDiff, MVT::i32); 9155 9156 unsigned SHLIntrinsicsID = 0; 9157 unsigned SRAIntrinsicsID = 0; 9158 switch (VT.getSimpleVT().SimpleTy) { 9159 default: 9160 return SDValue(); 9161 case MVT::v2i64: { 9162 SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_q; 9163 SRAIntrinsicsID = 0; 9164 break; 9165 } 9166 case MVT::v4i32: { 9167 SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_d; 9168 SRAIntrinsicsID = Intrinsic::x86_sse2_psrai_d; 9169 break; 9170 } 9171 case MVT::v8i16: { 9172 SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_w; 9173 SRAIntrinsicsID = Intrinsic::x86_sse2_psrai_w; 9174 break; 9175 } 9176 } 9177 9178 SDValue Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9179 DAG.getConstant(SHLIntrinsicsID, MVT::i32), 9180 Node->getOperand(0), ShAmt); 9181 9182 // In case of 1 bit sext, no need to shr 9183 if (ExtraVT.getScalarType().getSizeInBits() == 1) return Tmp1; 9184 9185 if (SRAIntrinsicsID) { 9186 Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9187 DAG.getConstant(SRAIntrinsicsID, MVT::i32), 9188 Tmp1, ShAmt); 9189 } 9190 return Tmp1; 9191 } 9192 9193 return SDValue(); 9194} 9195 9196 9197SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{ 9198 DebugLoc dl = Op.getDebugLoc(); 9199 9200 // Go ahead and emit the fence on x86-64 even if we asked for no-sse2. 9201 // There isn't any reason to disable it if the target processor supports it. 9202 if (!Subtarget->hasSSE2() && !Subtarget->is64Bit()) { 9203 SDValue Chain = Op.getOperand(0); 9204 SDValue Zero = DAG.getConstant(0, MVT::i32); 9205 SDValue Ops[] = { 9206 DAG.getRegister(X86::ESP, MVT::i32), // Base 9207 DAG.getTargetConstant(1, MVT::i8), // Scale 9208 DAG.getRegister(0, MVT::i32), // Index 9209 DAG.getTargetConstant(0, MVT::i32), // Disp 9210 DAG.getRegister(0, MVT::i32), // Segment. 9211 Zero, 9212 Chain 9213 }; 9214 SDNode *Res = 9215 DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops, 9216 array_lengthof(Ops)); 9217 return SDValue(Res, 0); 9218 } 9219 9220 unsigned isDev = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue(); 9221 if (!isDev) 9222 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); 9223 9224 unsigned Op1 = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 9225 unsigned Op2 = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 9226 unsigned Op3 = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 9227 unsigned Op4 = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 9228 9229 // def : Pat<(membarrier (i8 0), (i8 0), (i8 0), (i8 1), (i8 1)), (SFENCE)>; 9230 if (!Op1 && !Op2 && !Op3 && Op4) 9231 return DAG.getNode(X86ISD::SFENCE, dl, MVT::Other, Op.getOperand(0)); 9232 9233 // def : Pat<(membarrier (i8 1), (i8 0), (i8 0), (i8 0), (i8 1)), (LFENCE)>; 9234 if (Op1 && !Op2 && !Op3 && !Op4) 9235 return DAG.getNode(X86ISD::LFENCE, dl, MVT::Other, Op.getOperand(0)); 9236 9237 // def : Pat<(membarrier (i8 imm), (i8 imm), (i8 imm), (i8 imm), (i8 1)), 9238 // (MFENCE)>; 9239 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); 9240} 9241 9242SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const { 9243 EVT T = Op.getValueType(); 9244 DebugLoc DL = Op.getDebugLoc(); 9245 unsigned Reg = 0; 9246 unsigned size = 0; 9247 switch(T.getSimpleVT().SimpleTy) { 9248 default: 9249 assert(false && "Invalid value type!"); 9250 case MVT::i8: Reg = X86::AL; size = 1; break; 9251 case MVT::i16: Reg = X86::AX; size = 2; break; 9252 case MVT::i32: Reg = X86::EAX; size = 4; break; 9253 case MVT::i64: 9254 assert(Subtarget->is64Bit() && "Node not type legal!"); 9255 Reg = X86::RAX; size = 8; 9256 break; 9257 } 9258 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg, 9259 Op.getOperand(2), SDValue()); 9260 SDValue Ops[] = { cpIn.getValue(0), 9261 Op.getOperand(1), 9262 Op.getOperand(3), 9263 DAG.getTargetConstant(size, MVT::i8), 9264 cpIn.getValue(1) }; 9265 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 9266 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand(); 9267 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, 9268 Ops, 5, T, MMO); 9269 SDValue cpOut = 9270 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); 9271 return cpOut; 9272} 9273 9274SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op, 9275 SelectionDAG &DAG) const { 9276 assert(Subtarget->is64Bit() && "Result not type legalized?"); 9277 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 9278 SDValue TheChain = Op.getOperand(0); 9279 DebugLoc dl = Op.getDebugLoc(); 9280 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 9281 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1)); 9282 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64, 9283 rax.getValue(2)); 9284 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx, 9285 DAG.getConstant(32, MVT::i8)); 9286 SDValue Ops[] = { 9287 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp), 9288 rdx.getValue(1) 9289 }; 9290 return DAG.getMergeValues(Ops, 2, dl); 9291} 9292 9293SDValue X86TargetLowering::LowerBITCAST(SDValue Op, 9294 SelectionDAG &DAG) const { 9295 EVT SrcVT = Op.getOperand(0).getValueType(); 9296 EVT DstVT = Op.getValueType(); 9297 assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() && 9298 Subtarget->hasMMX() && "Unexpected custom BITCAST"); 9299 assert((DstVT == MVT::i64 || 9300 (DstVT.isVector() && DstVT.getSizeInBits()==64)) && 9301 "Unexpected custom BITCAST"); 9302 // i64 <=> MMX conversions are Legal. 9303 if (SrcVT==MVT::i64 && DstVT.isVector()) 9304 return Op; 9305 if (DstVT==MVT::i64 && SrcVT.isVector()) 9306 return Op; 9307 // MMX <=> MMX conversions are Legal. 9308 if (SrcVT.isVector() && DstVT.isVector()) 9309 return Op; 9310 // All other conversions need to be expanded. 9311 return SDValue(); 9312} 9313 9314SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const { 9315 SDNode *Node = Op.getNode(); 9316 DebugLoc dl = Node->getDebugLoc(); 9317 EVT T = Node->getValueType(0); 9318 SDValue negOp = DAG.getNode(ISD::SUB, dl, T, 9319 DAG.getConstant(0, T), Node->getOperand(2)); 9320 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, 9321 cast<AtomicSDNode>(Node)->getMemoryVT(), 9322 Node->getOperand(0), 9323 Node->getOperand(1), negOp, 9324 cast<AtomicSDNode>(Node)->getSrcValue(), 9325 cast<AtomicSDNode>(Node)->getAlignment()); 9326} 9327 9328static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) { 9329 EVT VT = Op.getNode()->getValueType(0); 9330 9331 // Let legalize expand this if it isn't a legal type yet. 9332 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 9333 return SDValue(); 9334 9335 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 9336 9337 unsigned Opc; 9338 bool ExtraOp = false; 9339 switch (Op.getOpcode()) { 9340 default: assert(0 && "Invalid code"); 9341 case ISD::ADDC: Opc = X86ISD::ADD; break; 9342 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break; 9343 case ISD::SUBC: Opc = X86ISD::SUB; break; 9344 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break; 9345 } 9346 9347 if (!ExtraOp) 9348 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), 9349 Op.getOperand(1)); 9350 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), 9351 Op.getOperand(1), Op.getOperand(2)); 9352} 9353 9354/// LowerOperation - Provide custom lowering hooks for some operations. 9355/// 9356SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 9357 switch (Op.getOpcode()) { 9358 default: llvm_unreachable("Should not custom lower this!"); 9359 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG); 9360 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op,DAG); 9361 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 9362 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); 9363 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 9364 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 9365 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 9366 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 9367 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 9368 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 9369 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); 9370 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 9371 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 9372 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 9373 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 9374 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 9375 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 9376 case ISD::SHL_PARTS: 9377 case ISD::SRA_PARTS: 9378 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG); 9379 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 9380 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 9381 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 9382 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); 9383 case ISD::FABS: return LowerFABS(Op, DAG); 9384 case ISD::FNEG: return LowerFNEG(Op, DAG); 9385 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 9386 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG); 9387 case ISD::SETCC: return LowerSETCC(Op, DAG); 9388 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 9389 case ISD::SELECT: return LowerSELECT(Op, DAG); 9390 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 9391 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 9392 case ISD::VASTART: return LowerVASTART(Op, DAG); 9393 case ISD::VAARG: return LowerVAARG(Op, DAG); 9394 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 9395 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 9396 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 9397 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 9398 case ISD::FRAME_TO_ARGS_OFFSET: 9399 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 9400 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 9401 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 9402 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 9403 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 9404 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 9405 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 9406 case ISD::MUL: return LowerMUL_V2I64(Op, DAG); 9407 case ISD::SRA: 9408 case ISD::SRL: 9409 case ISD::SHL: return LowerShift(Op, DAG); 9410 case ISD::SADDO: 9411 case ISD::UADDO: 9412 case ISD::SSUBO: 9413 case ISD::USUBO: 9414 case ISD::SMULO: 9415 case ISD::UMULO: return LowerXALUO(Op, DAG); 9416 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); 9417 case ISD::BITCAST: return LowerBITCAST(Op, DAG); 9418 case ISD::ADDC: 9419 case ISD::ADDE: 9420 case ISD::SUBC: 9421 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); 9422 } 9423} 9424 9425void X86TargetLowering:: 9426ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results, 9427 SelectionDAG &DAG, unsigned NewOp) const { 9428 EVT T = Node->getValueType(0); 9429 DebugLoc dl = Node->getDebugLoc(); 9430 assert (T == MVT::i64 && "Only know how to expand i64 atomics"); 9431 9432 SDValue Chain = Node->getOperand(0); 9433 SDValue In1 = Node->getOperand(1); 9434 SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 9435 Node->getOperand(2), DAG.getIntPtrConstant(0)); 9436 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 9437 Node->getOperand(2), DAG.getIntPtrConstant(1)); 9438 SDValue Ops[] = { Chain, In1, In2L, In2H }; 9439 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 9440 SDValue Result = 9441 DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, 9442 cast<MemSDNode>(Node)->getMemOperand()); 9443 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; 9444 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); 9445 Results.push_back(Result.getValue(2)); 9446} 9447 9448/// ReplaceNodeResults - Replace a node with an illegal result type 9449/// with a new node built out of custom code. 9450void X86TargetLowering::ReplaceNodeResults(SDNode *N, 9451 SmallVectorImpl<SDValue>&Results, 9452 SelectionDAG &DAG) const { 9453 DebugLoc dl = N->getDebugLoc(); 9454 switch (N->getOpcode()) { 9455 default: 9456 assert(false && "Do not know how to custom type legalize this operation!"); 9457 return; 9458 case ISD::SIGN_EXTEND_INREG: 9459 case ISD::ADDC: 9460 case ISD::ADDE: 9461 case ISD::SUBC: 9462 case ISD::SUBE: 9463 // We don't want to expand or promote these. 9464 return; 9465 case ISD::FP_TO_SINT: { 9466 std::pair<SDValue,SDValue> Vals = 9467 FP_TO_INTHelper(SDValue(N, 0), DAG, true); 9468 SDValue FIST = Vals.first, StackSlot = Vals.second; 9469 if (FIST.getNode() != 0) { 9470 EVT VT = N->getValueType(0); 9471 // Return a load from the stack slot. 9472 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, 9473 MachinePointerInfo(), false, false, 0)); 9474 } 9475 return; 9476 } 9477 case ISD::READCYCLECOUNTER: { 9478 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 9479 SDValue TheChain = N->getOperand(0); 9480 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 9481 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32, 9482 rd.getValue(1)); 9483 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32, 9484 eax.getValue(2)); 9485 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 9486 SDValue Ops[] = { eax, edx }; 9487 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2)); 9488 Results.push_back(edx.getValue(1)); 9489 return; 9490 } 9491 case ISD::ATOMIC_CMP_SWAP: { 9492 EVT T = N->getValueType(0); 9493 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 9494 SDValue cpInL, cpInH; 9495 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), 9496 DAG.getConstant(0, MVT::i32)); 9497 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), 9498 DAG.getConstant(1, MVT::i32)); 9499 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue()); 9500 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH, 9501 cpInL.getValue(1)); 9502 SDValue swapInL, swapInH; 9503 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), 9504 DAG.getConstant(0, MVT::i32)); 9505 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), 9506 DAG.getConstant(1, MVT::i32)); 9507 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL, 9508 cpInH.getValue(1)); 9509 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH, 9510 swapInL.getValue(1)); 9511 SDValue Ops[] = { swapInH.getValue(0), 9512 N->getOperand(1), 9513 swapInH.getValue(1) }; 9514 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 9515 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand(); 9516 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, 9517 Ops, 3, T, MMO); 9518 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX, 9519 MVT::i32, Result.getValue(1)); 9520 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX, 9521 MVT::i32, cpOutL.getValue(2)); 9522 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 9523 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); 9524 Results.push_back(cpOutH.getValue(1)); 9525 return; 9526 } 9527 case ISD::ATOMIC_LOAD_ADD: 9528 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG); 9529 return; 9530 case ISD::ATOMIC_LOAD_AND: 9531 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG); 9532 return; 9533 case ISD::ATOMIC_LOAD_NAND: 9534 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG); 9535 return; 9536 case ISD::ATOMIC_LOAD_OR: 9537 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG); 9538 return; 9539 case ISD::ATOMIC_LOAD_SUB: 9540 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG); 9541 return; 9542 case ISD::ATOMIC_LOAD_XOR: 9543 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG); 9544 return; 9545 case ISD::ATOMIC_SWAP: 9546 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG); 9547 return; 9548 } 9549} 9550 9551const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 9552 switch (Opcode) { 9553 default: return NULL; 9554 case X86ISD::BSF: return "X86ISD::BSF"; 9555 case X86ISD::BSR: return "X86ISD::BSR"; 9556 case X86ISD::SHLD: return "X86ISD::SHLD"; 9557 case X86ISD::SHRD: return "X86ISD::SHRD"; 9558 case X86ISD::FAND: return "X86ISD::FAND"; 9559 case X86ISD::FOR: return "X86ISD::FOR"; 9560 case X86ISD::FXOR: return "X86ISD::FXOR"; 9561 case X86ISD::FSRL: return "X86ISD::FSRL"; 9562 case X86ISD::FILD: return "X86ISD::FILD"; 9563 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 9564 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 9565 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 9566 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 9567 case X86ISD::FLD: return "X86ISD::FLD"; 9568 case X86ISD::FST: return "X86ISD::FST"; 9569 case X86ISD::CALL: return "X86ISD::CALL"; 9570 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 9571 case X86ISD::BT: return "X86ISD::BT"; 9572 case X86ISD::CMP: return "X86ISD::CMP"; 9573 case X86ISD::COMI: return "X86ISD::COMI"; 9574 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 9575 case X86ISD::SETCC: return "X86ISD::SETCC"; 9576 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY"; 9577 case X86ISD::FSETCCsd: return "X86ISD::FSETCCsd"; 9578 case X86ISD::FSETCCss: return "X86ISD::FSETCCss"; 9579 case X86ISD::CMOV: return "X86ISD::CMOV"; 9580 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 9581 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 9582 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 9583 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 9584 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 9585 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 9586 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP"; 9587 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 9588 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 9589 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 9590 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 9591 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 9592 case X86ISD::PSHUFB: return "X86ISD::PSHUFB"; 9593 case X86ISD::ANDNP: return "X86ISD::ANDNP"; 9594 case X86ISD::PSIGNB: return "X86ISD::PSIGNB"; 9595 case X86ISD::PSIGNW: return "X86ISD::PSIGNW"; 9596 case X86ISD::PSIGND: return "X86ISD::PSIGND"; 9597 case X86ISD::PBLENDVB: return "X86ISD::PBLENDVB"; 9598 case X86ISD::FMAX: return "X86ISD::FMAX"; 9599 case X86ISD::FMIN: return "X86ISD::FMIN"; 9600 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 9601 case X86ISD::FRCP: return "X86ISD::FRCP"; 9602 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 9603 case X86ISD::TLSCALL: return "X86ISD::TLSCALL"; 9604 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 9605 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 9606 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 9607 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 9608 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 9609 case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG"; 9610 case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG"; 9611 case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG"; 9612 case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG"; 9613 case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG"; 9614 case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG"; 9615 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 9616 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 9617 case X86ISD::VSHL: return "X86ISD::VSHL"; 9618 case X86ISD::VSRL: return "X86ISD::VSRL"; 9619 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 9620 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 9621 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 9622 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 9623 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 9624 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 9625 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 9626 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 9627 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 9628 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 9629 case X86ISD::ADD: return "X86ISD::ADD"; 9630 case X86ISD::SUB: return "X86ISD::SUB"; 9631 case X86ISD::ADC: return "X86ISD::ADC"; 9632 case X86ISD::SBB: return "X86ISD::SBB"; 9633 case X86ISD::SMUL: return "X86ISD::SMUL"; 9634 case X86ISD::UMUL: return "X86ISD::UMUL"; 9635 case X86ISD::INC: return "X86ISD::INC"; 9636 case X86ISD::DEC: return "X86ISD::DEC"; 9637 case X86ISD::OR: return "X86ISD::OR"; 9638 case X86ISD::XOR: return "X86ISD::XOR"; 9639 case X86ISD::AND: return "X86ISD::AND"; 9640 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM"; 9641 case X86ISD::PTEST: return "X86ISD::PTEST"; 9642 case X86ISD::TESTP: return "X86ISD::TESTP"; 9643 case X86ISD::PALIGN: return "X86ISD::PALIGN"; 9644 case X86ISD::PSHUFD: return "X86ISD::PSHUFD"; 9645 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW"; 9646 case X86ISD::PSHUFHW_LD: return "X86ISD::PSHUFHW_LD"; 9647 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW"; 9648 case X86ISD::PSHUFLW_LD: return "X86ISD::PSHUFLW_LD"; 9649 case X86ISD::SHUFPS: return "X86ISD::SHUFPS"; 9650 case X86ISD::SHUFPD: return "X86ISD::SHUFPD"; 9651 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS"; 9652 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD"; 9653 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS"; 9654 case X86ISD::MOVHLPD: return "X86ISD::MOVHLPD"; 9655 case X86ISD::MOVLPS: return "X86ISD::MOVLPS"; 9656 case X86ISD::MOVLPD: return "X86ISD::MOVLPD"; 9657 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP"; 9658 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP"; 9659 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP"; 9660 case X86ISD::MOVSHDUP_LD: return "X86ISD::MOVSHDUP_LD"; 9661 case X86ISD::MOVSLDUP_LD: return "X86ISD::MOVSLDUP_LD"; 9662 case X86ISD::MOVSD: return "X86ISD::MOVSD"; 9663 case X86ISD::MOVSS: return "X86ISD::MOVSS"; 9664 case X86ISD::UNPCKLPS: return "X86ISD::UNPCKLPS"; 9665 case X86ISD::UNPCKLPD: return "X86ISD::UNPCKLPD"; 9666 case X86ISD::VUNPCKLPDY: return "X86ISD::VUNPCKLPDY"; 9667 case X86ISD::UNPCKHPS: return "X86ISD::UNPCKHPS"; 9668 case X86ISD::UNPCKHPD: return "X86ISD::UNPCKHPD"; 9669 case X86ISD::PUNPCKLBW: return "X86ISD::PUNPCKLBW"; 9670 case X86ISD::PUNPCKLWD: return "X86ISD::PUNPCKLWD"; 9671 case X86ISD::PUNPCKLDQ: return "X86ISD::PUNPCKLDQ"; 9672 case X86ISD::PUNPCKLQDQ: return "X86ISD::PUNPCKLQDQ"; 9673 case X86ISD::PUNPCKHBW: return "X86ISD::PUNPCKHBW"; 9674 case X86ISD::PUNPCKHWD: return "X86ISD::PUNPCKHWD"; 9675 case X86ISD::PUNPCKHDQ: return "X86ISD::PUNPCKHDQ"; 9676 case X86ISD::PUNPCKHQDQ: return "X86ISD::PUNPCKHQDQ"; 9677 case X86ISD::VPERMIL: return "X86ISD::VPERMIL"; 9678 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS"; 9679 case X86ISD::VAARG_64: return "X86ISD::VAARG_64"; 9680 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA"; 9681 } 9682} 9683 9684// isLegalAddressingMode - Return true if the addressing mode represented 9685// by AM is legal for this target, for a load/store of the specified type. 9686bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 9687 Type *Ty) const { 9688 // X86 supports extremely general addressing modes. 9689 CodeModel::Model M = getTargetMachine().getCodeModel(); 9690 Reloc::Model R = getTargetMachine().getRelocationModel(); 9691 9692 // X86 allows a sign-extended 32-bit immediate field as a displacement. 9693 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL)) 9694 return false; 9695 9696 if (AM.BaseGV) { 9697 unsigned GVFlags = 9698 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine()); 9699 9700 // If a reference to this global requires an extra load, we can't fold it. 9701 if (isGlobalStubReference(GVFlags)) 9702 return false; 9703 9704 // If BaseGV requires a register for the PIC base, we cannot also have a 9705 // BaseReg specified. 9706 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags)) 9707 return false; 9708 9709 // If lower 4G is not available, then we must use rip-relative addressing. 9710 if ((M != CodeModel::Small || R != Reloc::Static) && 9711 Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1)) 9712 return false; 9713 } 9714 9715 switch (AM.Scale) { 9716 case 0: 9717 case 1: 9718 case 2: 9719 case 4: 9720 case 8: 9721 // These scales always work. 9722 break; 9723 case 3: 9724 case 5: 9725 case 9: 9726 // These scales are formed with basereg+scalereg. Only accept if there is 9727 // no basereg yet. 9728 if (AM.HasBaseReg) 9729 return false; 9730 break; 9731 default: // Other stuff never works. 9732 return false; 9733 } 9734 9735 return true; 9736} 9737 9738 9739bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 9740 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 9741 return false; 9742 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 9743 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 9744 if (NumBits1 <= NumBits2) 9745 return false; 9746 return true; 9747} 9748 9749bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 9750 if (!VT1.isInteger() || !VT2.isInteger()) 9751 return false; 9752 unsigned NumBits1 = VT1.getSizeInBits(); 9753 unsigned NumBits2 = VT2.getSizeInBits(); 9754 if (NumBits1 <= NumBits2) 9755 return false; 9756 return true; 9757} 9758 9759bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const { 9760 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 9761 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit(); 9762} 9763 9764bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const { 9765 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 9766 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); 9767} 9768 9769bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const { 9770 // i16 instructions are longer (0x66 prefix) and potentially slower. 9771 return !(VT1 == MVT::i32 && VT2 == MVT::i16); 9772} 9773 9774/// isShuffleMaskLegal - Targets can use this to indicate that they only 9775/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 9776/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 9777/// are assumed to be legal. 9778bool 9779X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M, 9780 EVT VT) const { 9781 // Very little shuffling can be done for 64-bit vectors right now. 9782 if (VT.getSizeInBits() == 64) 9783 return isPALIGNRMask(M, VT, Subtarget->hasSSSE3()); 9784 9785 // FIXME: pshufb, blends, shifts. 9786 return (VT.getVectorNumElements() == 2 || 9787 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || 9788 isMOVLMask(M, VT) || 9789 isSHUFPMask(M, VT) || 9790 isPSHUFDMask(M, VT) || 9791 isPSHUFHWMask(M, VT) || 9792 isPSHUFLWMask(M, VT) || 9793 isPALIGNRMask(M, VT, Subtarget->hasSSSE3()) || 9794 isUNPCKLMask(M, VT) || 9795 isUNPCKHMask(M, VT) || 9796 isUNPCKL_v_undef_Mask(M, VT) || 9797 isUNPCKH_v_undef_Mask(M, VT)); 9798} 9799 9800bool 9801X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask, 9802 EVT VT) const { 9803 unsigned NumElts = VT.getVectorNumElements(); 9804 // FIXME: This collection of masks seems suspect. 9805 if (NumElts == 2) 9806 return true; 9807 if (NumElts == 4 && VT.getSizeInBits() == 128) { 9808 return (isMOVLMask(Mask, VT) || 9809 isCommutedMOVLMask(Mask, VT, true) || 9810 isSHUFPMask(Mask, VT) || 9811 isCommutedSHUFPMask(Mask, VT)); 9812 } 9813 return false; 9814} 9815 9816//===----------------------------------------------------------------------===// 9817// X86 Scheduler Hooks 9818//===----------------------------------------------------------------------===// 9819 9820// private utility function 9821MachineBasicBlock * 9822X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 9823 MachineBasicBlock *MBB, 9824 unsigned regOpc, 9825 unsigned immOpc, 9826 unsigned LoadOpc, 9827 unsigned CXchgOpc, 9828 unsigned notOpc, 9829 unsigned EAXreg, 9830 TargetRegisterClass *RC, 9831 bool invSrc) const { 9832 // For the atomic bitwise operator, we generate 9833 // thisMBB: 9834 // newMBB: 9835 // ld t1 = [bitinstr.addr] 9836 // op t2 = t1, [bitinstr.val] 9837 // mov EAX = t1 9838 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 9839 // bz newMBB 9840 // fallthrough -->nextMBB 9841 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 9842 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 9843 MachineFunction::iterator MBBIter = MBB; 9844 ++MBBIter; 9845 9846 /// First build the CFG 9847 MachineFunction *F = MBB->getParent(); 9848 MachineBasicBlock *thisMBB = MBB; 9849 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 9850 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 9851 F->insert(MBBIter, newMBB); 9852 F->insert(MBBIter, nextMBB); 9853 9854 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 9855 nextMBB->splice(nextMBB->begin(), thisMBB, 9856 llvm::next(MachineBasicBlock::iterator(bInstr)), 9857 thisMBB->end()); 9858 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 9859 9860 // Update thisMBB to fall through to newMBB 9861 thisMBB->addSuccessor(newMBB); 9862 9863 // newMBB jumps to itself and fall through to nextMBB 9864 newMBB->addSuccessor(nextMBB); 9865 newMBB->addSuccessor(newMBB); 9866 9867 // Insert instructions into newMBB based on incoming instruction 9868 assert(bInstr->getNumOperands() < X86::AddrNumOperands + 4 && 9869 "unexpected number of operands"); 9870 DebugLoc dl = bInstr->getDebugLoc(); 9871 MachineOperand& destOper = bInstr->getOperand(0); 9872 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 9873 int numArgs = bInstr->getNumOperands() - 1; 9874 for (int i=0; i < numArgs; ++i) 9875 argOpers[i] = &bInstr->getOperand(i+1); 9876 9877 // x86 address has 4 operands: base, index, scale, and displacement 9878 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 9879 int valArgIndx = lastAddrIndx + 1; 9880 9881 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 9882 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1); 9883 for (int i=0; i <= lastAddrIndx; ++i) 9884 (*MIB).addOperand(*argOpers[i]); 9885 9886 unsigned tt = F->getRegInfo().createVirtualRegister(RC); 9887 if (invSrc) { 9888 MIB = BuildMI(newMBB, dl, TII->get(notOpc), tt).addReg(t1); 9889 } 9890 else 9891 tt = t1; 9892 9893 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 9894 assert((argOpers[valArgIndx]->isReg() || 9895 argOpers[valArgIndx]->isImm()) && 9896 "invalid operand"); 9897 if (argOpers[valArgIndx]->isReg()) 9898 MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2); 9899 else 9900 MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2); 9901 MIB.addReg(tt); 9902 (*MIB).addOperand(*argOpers[valArgIndx]); 9903 9904 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), EAXreg); 9905 MIB.addReg(t1); 9906 9907 MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc)); 9908 for (int i=0; i <= lastAddrIndx; ++i) 9909 (*MIB).addOperand(*argOpers[i]); 9910 MIB.addReg(t2); 9911 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 9912 (*MIB).setMemRefs(bInstr->memoperands_begin(), 9913 bInstr->memoperands_end()); 9914 9915 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg()); 9916 MIB.addReg(EAXreg); 9917 9918 // insert branch 9919 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 9920 9921 bInstr->eraseFromParent(); // The pseudo instruction is gone now. 9922 return nextMBB; 9923} 9924 9925// private utility function: 64 bit atomics on 32 bit host. 9926MachineBasicBlock * 9927X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr, 9928 MachineBasicBlock *MBB, 9929 unsigned regOpcL, 9930 unsigned regOpcH, 9931 unsigned immOpcL, 9932 unsigned immOpcH, 9933 bool invSrc) const { 9934 // For the atomic bitwise operator, we generate 9935 // thisMBB (instructions are in pairs, except cmpxchg8b) 9936 // ld t1,t2 = [bitinstr.addr] 9937 // newMBB: 9938 // out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4) 9939 // op t5, t6 <- out1, out2, [bitinstr.val] 9940 // (for SWAP, substitute: mov t5, t6 <- [bitinstr.val]) 9941 // mov ECX, EBX <- t5, t6 9942 // mov EAX, EDX <- t1, t2 9943 // cmpxchg8b [bitinstr.addr] [EAX, EDX, EBX, ECX implicit] 9944 // mov t3, t4 <- EAX, EDX 9945 // bz newMBB 9946 // result in out1, out2 9947 // fallthrough -->nextMBB 9948 9949 const TargetRegisterClass *RC = X86::GR32RegisterClass; 9950 const unsigned LoadOpc = X86::MOV32rm; 9951 const unsigned NotOpc = X86::NOT32r; 9952 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 9953 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 9954 MachineFunction::iterator MBBIter = MBB; 9955 ++MBBIter; 9956 9957 /// First build the CFG 9958 MachineFunction *F = MBB->getParent(); 9959 MachineBasicBlock *thisMBB = MBB; 9960 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 9961 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 9962 F->insert(MBBIter, newMBB); 9963 F->insert(MBBIter, nextMBB); 9964 9965 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 9966 nextMBB->splice(nextMBB->begin(), thisMBB, 9967 llvm::next(MachineBasicBlock::iterator(bInstr)), 9968 thisMBB->end()); 9969 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 9970 9971 // Update thisMBB to fall through to newMBB 9972 thisMBB->addSuccessor(newMBB); 9973 9974 // newMBB jumps to itself and fall through to nextMBB 9975 newMBB->addSuccessor(nextMBB); 9976 newMBB->addSuccessor(newMBB); 9977 9978 DebugLoc dl = bInstr->getDebugLoc(); 9979 // Insert instructions into newMBB based on incoming instruction 9980 // There are 8 "real" operands plus 9 implicit def/uses, ignored here. 9981 assert(bInstr->getNumOperands() < X86::AddrNumOperands + 14 && 9982 "unexpected number of operands"); 9983 MachineOperand& dest1Oper = bInstr->getOperand(0); 9984 MachineOperand& dest2Oper = bInstr->getOperand(1); 9985 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 9986 for (int i=0; i < 2 + X86::AddrNumOperands; ++i) { 9987 argOpers[i] = &bInstr->getOperand(i+2); 9988 9989 // We use some of the operands multiple times, so conservatively just 9990 // clear any kill flags that might be present. 9991 if (argOpers[i]->isReg() && argOpers[i]->isUse()) 9992 argOpers[i]->setIsKill(false); 9993 } 9994 9995 // x86 address has 5 operands: base, index, scale, displacement, and segment. 9996 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 9997 9998 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 9999 MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1); 10000 for (int i=0; i <= lastAddrIndx; ++i) 10001 (*MIB).addOperand(*argOpers[i]); 10002 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 10003 MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2); 10004 // add 4 to displacement. 10005 for (int i=0; i <= lastAddrIndx-2; ++i) 10006 (*MIB).addOperand(*argOpers[i]); 10007 MachineOperand newOp3 = *(argOpers[3]); 10008 if (newOp3.isImm()) 10009 newOp3.setImm(newOp3.getImm()+4); 10010 else 10011 newOp3.setOffset(newOp3.getOffset()+4); 10012 (*MIB).addOperand(newOp3); 10013 (*MIB).addOperand(*argOpers[lastAddrIndx]); 10014 10015 // t3/4 are defined later, at the bottom of the loop 10016 unsigned t3 = F->getRegInfo().createVirtualRegister(RC); 10017 unsigned t4 = F->getRegInfo().createVirtualRegister(RC); 10018 BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg()) 10019 .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB); 10020 BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg()) 10021 .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB); 10022 10023 // The subsequent operations should be using the destination registers of 10024 //the PHI instructions. 10025 if (invSrc) { 10026 t1 = F->getRegInfo().createVirtualRegister(RC); 10027 t2 = F->getRegInfo().createVirtualRegister(RC); 10028 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t1).addReg(dest1Oper.getReg()); 10029 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t2).addReg(dest2Oper.getReg()); 10030 } else { 10031 t1 = dest1Oper.getReg(); 10032 t2 = dest2Oper.getReg(); 10033 } 10034 10035 int valArgIndx = lastAddrIndx + 1; 10036 assert((argOpers[valArgIndx]->isReg() || 10037 argOpers[valArgIndx]->isImm()) && 10038 "invalid operand"); 10039 unsigned t5 = F->getRegInfo().createVirtualRegister(RC); 10040 unsigned t6 = F->getRegInfo().createVirtualRegister(RC); 10041 if (argOpers[valArgIndx]->isReg()) 10042 MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5); 10043 else 10044 MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5); 10045 if (regOpcL != X86::MOV32rr) 10046 MIB.addReg(t1); 10047 (*MIB).addOperand(*argOpers[valArgIndx]); 10048 assert(argOpers[valArgIndx + 1]->isReg() == 10049 argOpers[valArgIndx]->isReg()); 10050 assert(argOpers[valArgIndx + 1]->isImm() == 10051 argOpers[valArgIndx]->isImm()); 10052 if (argOpers[valArgIndx + 1]->isReg()) 10053 MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6); 10054 else 10055 MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6); 10056 if (regOpcH != X86::MOV32rr) 10057 MIB.addReg(t2); 10058 (*MIB).addOperand(*argOpers[valArgIndx + 1]); 10059 10060 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX); 10061 MIB.addReg(t1); 10062 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EDX); 10063 MIB.addReg(t2); 10064 10065 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EBX); 10066 MIB.addReg(t5); 10067 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::ECX); 10068 MIB.addReg(t6); 10069 10070 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B)); 10071 for (int i=0; i <= lastAddrIndx; ++i) 10072 (*MIB).addOperand(*argOpers[i]); 10073 10074 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 10075 (*MIB).setMemRefs(bInstr->memoperands_begin(), 10076 bInstr->memoperands_end()); 10077 10078 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t3); 10079 MIB.addReg(X86::EAX); 10080 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t4); 10081 MIB.addReg(X86::EDX); 10082 10083 // insert branch 10084 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 10085 10086 bInstr->eraseFromParent(); // The pseudo instruction is gone now. 10087 return nextMBB; 10088} 10089 10090// private utility function 10091MachineBasicBlock * 10092X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 10093 MachineBasicBlock *MBB, 10094 unsigned cmovOpc) const { 10095 // For the atomic min/max operator, we generate 10096 // thisMBB: 10097 // newMBB: 10098 // ld t1 = [min/max.addr] 10099 // mov t2 = [min/max.val] 10100 // cmp t1, t2 10101 // cmov[cond] t2 = t1 10102 // mov EAX = t1 10103 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 10104 // bz newMBB 10105 // fallthrough -->nextMBB 10106 // 10107 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10108 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 10109 MachineFunction::iterator MBBIter = MBB; 10110 ++MBBIter; 10111 10112 /// First build the CFG 10113 MachineFunction *F = MBB->getParent(); 10114 MachineBasicBlock *thisMBB = MBB; 10115 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 10116 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 10117 F->insert(MBBIter, newMBB); 10118 F->insert(MBBIter, nextMBB); 10119 10120 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 10121 nextMBB->splice(nextMBB->begin(), thisMBB, 10122 llvm::next(MachineBasicBlock::iterator(mInstr)), 10123 thisMBB->end()); 10124 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 10125 10126 // Update thisMBB to fall through to newMBB 10127 thisMBB->addSuccessor(newMBB); 10128 10129 // newMBB jumps to newMBB and fall through to nextMBB 10130 newMBB->addSuccessor(nextMBB); 10131 newMBB->addSuccessor(newMBB); 10132 10133 DebugLoc dl = mInstr->getDebugLoc(); 10134 // Insert instructions into newMBB based on incoming instruction 10135 assert(mInstr->getNumOperands() < X86::AddrNumOperands + 4 && 10136 "unexpected number of operands"); 10137 MachineOperand& destOper = mInstr->getOperand(0); 10138 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 10139 int numArgs = mInstr->getNumOperands() - 1; 10140 for (int i=0; i < numArgs; ++i) 10141 argOpers[i] = &mInstr->getOperand(i+1); 10142 10143 // x86 address has 4 operands: base, index, scale, and displacement 10144 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 10145 int valArgIndx = lastAddrIndx + 1; 10146 10147 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 10148 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1); 10149 for (int i=0; i <= lastAddrIndx; ++i) 10150 (*MIB).addOperand(*argOpers[i]); 10151 10152 // We only support register and immediate values 10153 assert((argOpers[valArgIndx]->isReg() || 10154 argOpers[valArgIndx]->isImm()) && 10155 "invalid operand"); 10156 10157 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 10158 if (argOpers[valArgIndx]->isReg()) 10159 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t2); 10160 else 10161 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2); 10162 (*MIB).addOperand(*argOpers[valArgIndx]); 10163 10164 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX); 10165 MIB.addReg(t1); 10166 10167 MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr)); 10168 MIB.addReg(t1); 10169 MIB.addReg(t2); 10170 10171 // Generate movc 10172 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 10173 MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3); 10174 MIB.addReg(t2); 10175 MIB.addReg(t1); 10176 10177 // Cmp and exchange if none has modified the memory location 10178 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32)); 10179 for (int i=0; i <= lastAddrIndx; ++i) 10180 (*MIB).addOperand(*argOpers[i]); 10181 MIB.addReg(t3); 10182 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 10183 (*MIB).setMemRefs(mInstr->memoperands_begin(), 10184 mInstr->memoperands_end()); 10185 10186 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg()); 10187 MIB.addReg(X86::EAX); 10188 10189 // insert branch 10190 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 10191 10192 mInstr->eraseFromParent(); // The pseudo instruction is gone now. 10193 return nextMBB; 10194} 10195 10196// FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8 10197// or XMM0_V32I8 in AVX all of this code can be replaced with that 10198// in the .td file. 10199MachineBasicBlock * 10200X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB, 10201 unsigned numArgs, bool memArg) const { 10202 assert((Subtarget->hasSSE42() || Subtarget->hasAVX()) && 10203 "Target must have SSE4.2 or AVX features enabled"); 10204 10205 DebugLoc dl = MI->getDebugLoc(); 10206 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10207 unsigned Opc; 10208 if (!Subtarget->hasAVX()) { 10209 if (memArg) 10210 Opc = numArgs == 3 ? X86::PCMPISTRM128rm : X86::PCMPESTRM128rm; 10211 else 10212 Opc = numArgs == 3 ? X86::PCMPISTRM128rr : X86::PCMPESTRM128rr; 10213 } else { 10214 if (memArg) 10215 Opc = numArgs == 3 ? X86::VPCMPISTRM128rm : X86::VPCMPESTRM128rm; 10216 else 10217 Opc = numArgs == 3 ? X86::VPCMPISTRM128rr : X86::VPCMPESTRM128rr; 10218 } 10219 10220 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc)); 10221 for (unsigned i = 0; i < numArgs; ++i) { 10222 MachineOperand &Op = MI->getOperand(i+1); 10223 if (!(Op.isReg() && Op.isImplicit())) 10224 MIB.addOperand(Op); 10225 } 10226 BuildMI(*BB, MI, dl, TII->get(X86::MOVAPSrr), MI->getOperand(0).getReg()) 10227 .addReg(X86::XMM0); 10228 10229 MI->eraseFromParent(); 10230 return BB; 10231} 10232 10233MachineBasicBlock * 10234X86TargetLowering::EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB) const { 10235 DebugLoc dl = MI->getDebugLoc(); 10236 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10237 10238 // Address into RAX/EAX, other two args into ECX, EDX. 10239 unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r; 10240 unsigned MemReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; 10241 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(MemOpc), MemReg); 10242 for (int i = 0; i < X86::AddrNumOperands; ++i) 10243 MIB.addOperand(MI->getOperand(i)); 10244 10245 unsigned ValOps = X86::AddrNumOperands; 10246 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX) 10247 .addReg(MI->getOperand(ValOps).getReg()); 10248 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EDX) 10249 .addReg(MI->getOperand(ValOps+1).getReg()); 10250 10251 // The instruction doesn't actually take any operands though. 10252 BuildMI(*BB, MI, dl, TII->get(X86::MONITORrrr)); 10253 10254 MI->eraseFromParent(); // The pseudo is gone now. 10255 return BB; 10256} 10257 10258MachineBasicBlock * 10259X86TargetLowering::EmitMwait(MachineInstr *MI, MachineBasicBlock *BB) const { 10260 DebugLoc dl = MI->getDebugLoc(); 10261 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10262 10263 // First arg in ECX, the second in EAX. 10264 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX) 10265 .addReg(MI->getOperand(0).getReg()); 10266 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EAX) 10267 .addReg(MI->getOperand(1).getReg()); 10268 10269 // The instruction doesn't actually take any operands though. 10270 BuildMI(*BB, MI, dl, TII->get(X86::MWAITrr)); 10271 10272 MI->eraseFromParent(); // The pseudo is gone now. 10273 return BB; 10274} 10275 10276MachineBasicBlock * 10277X86TargetLowering::EmitVAARG64WithCustomInserter( 10278 MachineInstr *MI, 10279 MachineBasicBlock *MBB) const { 10280 // Emit va_arg instruction on X86-64. 10281 10282 // Operands to this pseudo-instruction: 10283 // 0 ) Output : destination address (reg) 10284 // 1-5) Input : va_list address (addr, i64mem) 10285 // 6 ) ArgSize : Size (in bytes) of vararg type 10286 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset 10287 // 8 ) Align : Alignment of type 10288 // 9 ) EFLAGS (implicit-def) 10289 10290 assert(MI->getNumOperands() == 10 && "VAARG_64 should have 10 operands!"); 10291 assert(X86::AddrNumOperands == 5 && "VAARG_64 assumes 5 address operands"); 10292 10293 unsigned DestReg = MI->getOperand(0).getReg(); 10294 MachineOperand &Base = MI->getOperand(1); 10295 MachineOperand &Scale = MI->getOperand(2); 10296 MachineOperand &Index = MI->getOperand(3); 10297 MachineOperand &Disp = MI->getOperand(4); 10298 MachineOperand &Segment = MI->getOperand(5); 10299 unsigned ArgSize = MI->getOperand(6).getImm(); 10300 unsigned ArgMode = MI->getOperand(7).getImm(); 10301 unsigned Align = MI->getOperand(8).getImm(); 10302 10303 // Memory Reference 10304 assert(MI->hasOneMemOperand() && "Expected VAARG_64 to have one memoperand"); 10305 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 10306 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 10307 10308 // Machine Information 10309 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10310 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 10311 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); 10312 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); 10313 DebugLoc DL = MI->getDebugLoc(); 10314 10315 // struct va_list { 10316 // i32 gp_offset 10317 // i32 fp_offset 10318 // i64 overflow_area (address) 10319 // i64 reg_save_area (address) 10320 // } 10321 // sizeof(va_list) = 24 10322 // alignment(va_list) = 8 10323 10324 unsigned TotalNumIntRegs = 6; 10325 unsigned TotalNumXMMRegs = 8; 10326 bool UseGPOffset = (ArgMode == 1); 10327 bool UseFPOffset = (ArgMode == 2); 10328 unsigned MaxOffset = TotalNumIntRegs * 8 + 10329 (UseFPOffset ? TotalNumXMMRegs * 16 : 0); 10330 10331 /* Align ArgSize to a multiple of 8 */ 10332 unsigned ArgSizeA8 = (ArgSize + 7) & ~7; 10333 bool NeedsAlign = (Align > 8); 10334 10335 MachineBasicBlock *thisMBB = MBB; 10336 MachineBasicBlock *overflowMBB; 10337 MachineBasicBlock *offsetMBB; 10338 MachineBasicBlock *endMBB; 10339 10340 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB 10341 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB 10342 unsigned OffsetReg = 0; 10343 10344 if (!UseGPOffset && !UseFPOffset) { 10345 // If we only pull from the overflow region, we don't create a branch. 10346 // We don't need to alter control flow. 10347 OffsetDestReg = 0; // unused 10348 OverflowDestReg = DestReg; 10349 10350 offsetMBB = NULL; 10351 overflowMBB = thisMBB; 10352 endMBB = thisMBB; 10353 } else { 10354 // First emit code to check if gp_offset (or fp_offset) is below the bound. 10355 // If so, pull the argument from reg_save_area. (branch to offsetMBB) 10356 // If not, pull from overflow_area. (branch to overflowMBB) 10357 // 10358 // thisMBB 10359 // | . 10360 // | . 10361 // offsetMBB overflowMBB 10362 // | . 10363 // | . 10364 // endMBB 10365 10366 // Registers for the PHI in endMBB 10367 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass); 10368 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass); 10369 10370 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 10371 MachineFunction *MF = MBB->getParent(); 10372 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB); 10373 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB); 10374 endMBB = MF->CreateMachineBasicBlock(LLVM_BB); 10375 10376 MachineFunction::iterator MBBIter = MBB; 10377 ++MBBIter; 10378 10379 // Insert the new basic blocks 10380 MF->insert(MBBIter, offsetMBB); 10381 MF->insert(MBBIter, overflowMBB); 10382 MF->insert(MBBIter, endMBB); 10383 10384 // Transfer the remainder of MBB and its successor edges to endMBB. 10385 endMBB->splice(endMBB->begin(), thisMBB, 10386 llvm::next(MachineBasicBlock::iterator(MI)), 10387 thisMBB->end()); 10388 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 10389 10390 // Make offsetMBB and overflowMBB successors of thisMBB 10391 thisMBB->addSuccessor(offsetMBB); 10392 thisMBB->addSuccessor(overflowMBB); 10393 10394 // endMBB is a successor of both offsetMBB and overflowMBB 10395 offsetMBB->addSuccessor(endMBB); 10396 overflowMBB->addSuccessor(endMBB); 10397 10398 // Load the offset value into a register 10399 OffsetReg = MRI.createVirtualRegister(OffsetRegClass); 10400 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg) 10401 .addOperand(Base) 10402 .addOperand(Scale) 10403 .addOperand(Index) 10404 .addDisp(Disp, UseFPOffset ? 4 : 0) 10405 .addOperand(Segment) 10406 .setMemRefs(MMOBegin, MMOEnd); 10407 10408 // Check if there is enough room left to pull this argument. 10409 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri)) 10410 .addReg(OffsetReg) 10411 .addImm(MaxOffset + 8 - ArgSizeA8); 10412 10413 // Branch to "overflowMBB" if offset >= max 10414 // Fall through to "offsetMBB" otherwise 10415 BuildMI(thisMBB, DL, TII->get(X86::GetCondBranchFromCond(X86::COND_AE))) 10416 .addMBB(overflowMBB); 10417 } 10418 10419 // In offsetMBB, emit code to use the reg_save_area. 10420 if (offsetMBB) { 10421 assert(OffsetReg != 0); 10422 10423 // Read the reg_save_area address. 10424 unsigned RegSaveReg = MRI.createVirtualRegister(AddrRegClass); 10425 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg) 10426 .addOperand(Base) 10427 .addOperand(Scale) 10428 .addOperand(Index) 10429 .addDisp(Disp, 16) 10430 .addOperand(Segment) 10431 .setMemRefs(MMOBegin, MMOEnd); 10432 10433 // Zero-extend the offset 10434 unsigned OffsetReg64 = MRI.createVirtualRegister(AddrRegClass); 10435 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64) 10436 .addImm(0) 10437 .addReg(OffsetReg) 10438 .addImm(X86::sub_32bit); 10439 10440 // Add the offset to the reg_save_area to get the final address. 10441 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg) 10442 .addReg(OffsetReg64) 10443 .addReg(RegSaveReg); 10444 10445 // Compute the offset for the next argument 10446 unsigned NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass); 10447 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg) 10448 .addReg(OffsetReg) 10449 .addImm(UseFPOffset ? 16 : 8); 10450 10451 // Store it back into the va_list. 10452 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr)) 10453 .addOperand(Base) 10454 .addOperand(Scale) 10455 .addOperand(Index) 10456 .addDisp(Disp, UseFPOffset ? 4 : 0) 10457 .addOperand(Segment) 10458 .addReg(NextOffsetReg) 10459 .setMemRefs(MMOBegin, MMOEnd); 10460 10461 // Jump to endMBB 10462 BuildMI(offsetMBB, DL, TII->get(X86::JMP_4)) 10463 .addMBB(endMBB); 10464 } 10465 10466 // 10467 // Emit code to use overflow area 10468 // 10469 10470 // Load the overflow_area address into a register. 10471 unsigned OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass); 10472 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg) 10473 .addOperand(Base) 10474 .addOperand(Scale) 10475 .addOperand(Index) 10476 .addDisp(Disp, 8) 10477 .addOperand(Segment) 10478 .setMemRefs(MMOBegin, MMOEnd); 10479 10480 // If we need to align it, do so. Otherwise, just copy the address 10481 // to OverflowDestReg. 10482 if (NeedsAlign) { 10483 // Align the overflow address 10484 assert((Align & (Align-1)) == 0 && "Alignment must be a power of 2"); 10485 unsigned TmpReg = MRI.createVirtualRegister(AddrRegClass); 10486 10487 // aligned_addr = (addr + (align-1)) & ~(align-1) 10488 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg) 10489 .addReg(OverflowAddrReg) 10490 .addImm(Align-1); 10491 10492 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg) 10493 .addReg(TmpReg) 10494 .addImm(~(uint64_t)(Align-1)); 10495 } else { 10496 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg) 10497 .addReg(OverflowAddrReg); 10498 } 10499 10500 // Compute the next overflow address after this argument. 10501 // (the overflow address should be kept 8-byte aligned) 10502 unsigned NextAddrReg = MRI.createVirtualRegister(AddrRegClass); 10503 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg) 10504 .addReg(OverflowDestReg) 10505 .addImm(ArgSizeA8); 10506 10507 // Store the new overflow address. 10508 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr)) 10509 .addOperand(Base) 10510 .addOperand(Scale) 10511 .addOperand(Index) 10512 .addDisp(Disp, 8) 10513 .addOperand(Segment) 10514 .addReg(NextAddrReg) 10515 .setMemRefs(MMOBegin, MMOEnd); 10516 10517 // If we branched, emit the PHI to the front of endMBB. 10518 if (offsetMBB) { 10519 BuildMI(*endMBB, endMBB->begin(), DL, 10520 TII->get(X86::PHI), DestReg) 10521 .addReg(OffsetDestReg).addMBB(offsetMBB) 10522 .addReg(OverflowDestReg).addMBB(overflowMBB); 10523 } 10524 10525 // Erase the pseudo instruction 10526 MI->eraseFromParent(); 10527 10528 return endMBB; 10529} 10530 10531MachineBasicBlock * 10532X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( 10533 MachineInstr *MI, 10534 MachineBasicBlock *MBB) const { 10535 // Emit code to save XMM registers to the stack. The ABI says that the 10536 // number of registers to save is given in %al, so it's theoretically 10537 // possible to do an indirect jump trick to avoid saving all of them, 10538 // however this code takes a simpler approach and just executes all 10539 // of the stores if %al is non-zero. It's less code, and it's probably 10540 // easier on the hardware branch predictor, and stores aren't all that 10541 // expensive anyway. 10542 10543 // Create the new basic blocks. One block contains all the XMM stores, 10544 // and one block is the final destination regardless of whether any 10545 // stores were performed. 10546 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 10547 MachineFunction *F = MBB->getParent(); 10548 MachineFunction::iterator MBBIter = MBB; 10549 ++MBBIter; 10550 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB); 10551 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB); 10552 F->insert(MBBIter, XMMSaveMBB); 10553 F->insert(MBBIter, EndMBB); 10554 10555 // Transfer the remainder of MBB and its successor edges to EndMBB. 10556 EndMBB->splice(EndMBB->begin(), MBB, 10557 llvm::next(MachineBasicBlock::iterator(MI)), 10558 MBB->end()); 10559 EndMBB->transferSuccessorsAndUpdatePHIs(MBB); 10560 10561 // The original block will now fall through to the XMM save block. 10562 MBB->addSuccessor(XMMSaveMBB); 10563 // The XMMSaveMBB will fall through to the end block. 10564 XMMSaveMBB->addSuccessor(EndMBB); 10565 10566 // Now add the instructions. 10567 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10568 DebugLoc DL = MI->getDebugLoc(); 10569 10570 unsigned CountReg = MI->getOperand(0).getReg(); 10571 int64_t RegSaveFrameIndex = MI->getOperand(1).getImm(); 10572 int64_t VarArgsFPOffset = MI->getOperand(2).getImm(); 10573 10574 if (!Subtarget->isTargetWin64()) { 10575 // If %al is 0, branch around the XMM save block. 10576 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg); 10577 BuildMI(MBB, DL, TII->get(X86::JE_4)).addMBB(EndMBB); 10578 MBB->addSuccessor(EndMBB); 10579 } 10580 10581 // In the XMM save block, save all the XMM argument registers. 10582 for (int i = 3, e = MI->getNumOperands(); i != e; ++i) { 10583 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset; 10584 MachineMemOperand *MMO = 10585 F->getMachineMemOperand( 10586 MachinePointerInfo::getFixedStack(RegSaveFrameIndex, Offset), 10587 MachineMemOperand::MOStore, 10588 /*Size=*/16, /*Align=*/16); 10589 BuildMI(XMMSaveMBB, DL, TII->get(X86::MOVAPSmr)) 10590 .addFrameIndex(RegSaveFrameIndex) 10591 .addImm(/*Scale=*/1) 10592 .addReg(/*IndexReg=*/0) 10593 .addImm(/*Disp=*/Offset) 10594 .addReg(/*Segment=*/0) 10595 .addReg(MI->getOperand(i).getReg()) 10596 .addMemOperand(MMO); 10597 } 10598 10599 MI->eraseFromParent(); // The pseudo instruction is gone now. 10600 10601 return EndMBB; 10602} 10603 10604MachineBasicBlock * 10605X86TargetLowering::EmitLoweredSelect(MachineInstr *MI, 10606 MachineBasicBlock *BB) const { 10607 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10608 DebugLoc DL = MI->getDebugLoc(); 10609 10610 // To "insert" a SELECT_CC instruction, we actually have to insert the 10611 // diamond control-flow pattern. The incoming instruction knows the 10612 // destination vreg to set, the condition code register to branch on, the 10613 // true/false values to select between, and a branch opcode to use. 10614 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10615 MachineFunction::iterator It = BB; 10616 ++It; 10617 10618 // thisMBB: 10619 // ... 10620 // TrueVal = ... 10621 // cmpTY ccX, r1, r2 10622 // bCC copy1MBB 10623 // fallthrough --> copy0MBB 10624 MachineBasicBlock *thisMBB = BB; 10625 MachineFunction *F = BB->getParent(); 10626 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 10627 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 10628 F->insert(It, copy0MBB); 10629 F->insert(It, sinkMBB); 10630 10631 // If the EFLAGS register isn't dead in the terminator, then claim that it's 10632 // live into the sink and copy blocks. 10633 const MachineFunction *MF = BB->getParent(); 10634 const TargetRegisterInfo *TRI = MF->getTarget().getRegisterInfo(); 10635 BitVector ReservedRegs = TRI->getReservedRegs(*MF); 10636 10637 for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) { 10638 const MachineOperand &MO = MI->getOperand(I); 10639 if (!MO.isReg() || !MO.isUse() || MO.isKill()) continue; 10640 unsigned Reg = MO.getReg(); 10641 if (Reg != X86::EFLAGS) continue; 10642 copy0MBB->addLiveIn(Reg); 10643 sinkMBB->addLiveIn(Reg); 10644 } 10645 10646 // Transfer the remainder of BB and its successor edges to sinkMBB. 10647 sinkMBB->splice(sinkMBB->begin(), BB, 10648 llvm::next(MachineBasicBlock::iterator(MI)), 10649 BB->end()); 10650 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 10651 10652 // Add the true and fallthrough blocks as its successors. 10653 BB->addSuccessor(copy0MBB); 10654 BB->addSuccessor(sinkMBB); 10655 10656 // Create the conditional branch instruction. 10657 unsigned Opc = 10658 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 10659 BuildMI(BB, DL, TII->get(Opc)).addMBB(sinkMBB); 10660 10661 // copy0MBB: 10662 // %FalseValue = ... 10663 // # fallthrough to sinkMBB 10664 copy0MBB->addSuccessor(sinkMBB); 10665 10666 // sinkMBB: 10667 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 10668 // ... 10669 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 10670 TII->get(X86::PHI), MI->getOperand(0).getReg()) 10671 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 10672 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 10673 10674 MI->eraseFromParent(); // The pseudo instruction is gone now. 10675 return sinkMBB; 10676} 10677 10678MachineBasicBlock * 10679X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI, 10680 MachineBasicBlock *BB) const { 10681 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10682 DebugLoc DL = MI->getDebugLoc(); 10683 10684 assert(!Subtarget->isTargetEnvMacho()); 10685 10686 // The lowering is pretty easy: we're just emitting the call to _alloca. The 10687 // non-trivial part is impdef of ESP. 10688 10689 if (Subtarget->isTargetWin64()) { 10690 if (Subtarget->isTargetCygMing()) { 10691 // ___chkstk(Mingw64): 10692 // Clobbers R10, R11, RAX and EFLAGS. 10693 // Updates RSP. 10694 BuildMI(*BB, MI, DL, TII->get(X86::W64ALLOCA)) 10695 .addExternalSymbol("___chkstk") 10696 .addReg(X86::RAX, RegState::Implicit) 10697 .addReg(X86::RSP, RegState::Implicit) 10698 .addReg(X86::RAX, RegState::Define | RegState::Implicit) 10699 .addReg(X86::RSP, RegState::Define | RegState::Implicit) 10700 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 10701 } else { 10702 // __chkstk(MSVCRT): does not update stack pointer. 10703 // Clobbers R10, R11 and EFLAGS. 10704 // FIXME: RAX(allocated size) might be reused and not killed. 10705 BuildMI(*BB, MI, DL, TII->get(X86::W64ALLOCA)) 10706 .addExternalSymbol("__chkstk") 10707 .addReg(X86::RAX, RegState::Implicit) 10708 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 10709 // RAX has the offset to subtracted from RSP. 10710 BuildMI(*BB, MI, DL, TII->get(X86::SUB64rr), X86::RSP) 10711 .addReg(X86::RSP) 10712 .addReg(X86::RAX); 10713 } 10714 } else { 10715 const char *StackProbeSymbol = 10716 Subtarget->isTargetWindows() ? "_chkstk" : "_alloca"; 10717 10718 BuildMI(*BB, MI, DL, TII->get(X86::CALLpcrel32)) 10719 .addExternalSymbol(StackProbeSymbol) 10720 .addReg(X86::EAX, RegState::Implicit) 10721 .addReg(X86::ESP, RegState::Implicit) 10722 .addReg(X86::EAX, RegState::Define | RegState::Implicit) 10723 .addReg(X86::ESP, RegState::Define | RegState::Implicit) 10724 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 10725 } 10726 10727 MI->eraseFromParent(); // The pseudo instruction is gone now. 10728 return BB; 10729} 10730 10731MachineBasicBlock * 10732X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI, 10733 MachineBasicBlock *BB) const { 10734 // This is pretty easy. We're taking the value that we received from 10735 // our load from the relocation, sticking it in either RDI (x86-64) 10736 // or EAX and doing an indirect call. The return value will then 10737 // be in the normal return register. 10738 const X86InstrInfo *TII 10739 = static_cast<const X86InstrInfo*>(getTargetMachine().getInstrInfo()); 10740 DebugLoc DL = MI->getDebugLoc(); 10741 MachineFunction *F = BB->getParent(); 10742 10743 assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?"); 10744 assert(MI->getOperand(3).isGlobal() && "This should be a global"); 10745 10746 if (Subtarget->is64Bit()) { 10747 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 10748 TII->get(X86::MOV64rm), X86::RDI) 10749 .addReg(X86::RIP) 10750 .addImm(0).addReg(0) 10751 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 10752 MI->getOperand(3).getTargetFlags()) 10753 .addReg(0); 10754 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m)); 10755 addDirectMem(MIB, X86::RDI); 10756 } else if (getTargetMachine().getRelocationModel() != Reloc::PIC_) { 10757 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 10758 TII->get(X86::MOV32rm), X86::EAX) 10759 .addReg(0) 10760 .addImm(0).addReg(0) 10761 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 10762 MI->getOperand(3).getTargetFlags()) 10763 .addReg(0); 10764 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m)); 10765 addDirectMem(MIB, X86::EAX); 10766 } else { 10767 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 10768 TII->get(X86::MOV32rm), X86::EAX) 10769 .addReg(TII->getGlobalBaseReg(F)) 10770 .addImm(0).addReg(0) 10771 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 10772 MI->getOperand(3).getTargetFlags()) 10773 .addReg(0); 10774 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m)); 10775 addDirectMem(MIB, X86::EAX); 10776 } 10777 10778 MI->eraseFromParent(); // The pseudo instruction is gone now. 10779 return BB; 10780} 10781 10782MachineBasicBlock * 10783X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 10784 MachineBasicBlock *BB) const { 10785 switch (MI->getOpcode()) { 10786 default: assert(false && "Unexpected instr type to insert"); 10787 case X86::TAILJMPd64: 10788 case X86::TAILJMPr64: 10789 case X86::TAILJMPm64: 10790 assert(!"TAILJMP64 would not be touched here."); 10791 case X86::TCRETURNdi64: 10792 case X86::TCRETURNri64: 10793 case X86::TCRETURNmi64: 10794 // Defs of TCRETURNxx64 has Win64's callee-saved registers, as subset. 10795 // On AMD64, additional defs should be added before register allocation. 10796 if (!Subtarget->isTargetWin64()) { 10797 MI->addRegisterDefined(X86::RSI); 10798 MI->addRegisterDefined(X86::RDI); 10799 MI->addRegisterDefined(X86::XMM6); 10800 MI->addRegisterDefined(X86::XMM7); 10801 MI->addRegisterDefined(X86::XMM8); 10802 MI->addRegisterDefined(X86::XMM9); 10803 MI->addRegisterDefined(X86::XMM10); 10804 MI->addRegisterDefined(X86::XMM11); 10805 MI->addRegisterDefined(X86::XMM12); 10806 MI->addRegisterDefined(X86::XMM13); 10807 MI->addRegisterDefined(X86::XMM14); 10808 MI->addRegisterDefined(X86::XMM15); 10809 } 10810 return BB; 10811 case X86::WIN_ALLOCA: 10812 return EmitLoweredWinAlloca(MI, BB); 10813 case X86::TLSCall_32: 10814 case X86::TLSCall_64: 10815 return EmitLoweredTLSCall(MI, BB); 10816 case X86::CMOV_GR8: 10817 case X86::CMOV_FR32: 10818 case X86::CMOV_FR64: 10819 case X86::CMOV_V4F32: 10820 case X86::CMOV_V2F64: 10821 case X86::CMOV_V2I64: 10822 case X86::CMOV_GR16: 10823 case X86::CMOV_GR32: 10824 case X86::CMOV_RFP32: 10825 case X86::CMOV_RFP64: 10826 case X86::CMOV_RFP80: 10827 return EmitLoweredSelect(MI, BB); 10828 10829 case X86::FP32_TO_INT16_IN_MEM: 10830 case X86::FP32_TO_INT32_IN_MEM: 10831 case X86::FP32_TO_INT64_IN_MEM: 10832 case X86::FP64_TO_INT16_IN_MEM: 10833 case X86::FP64_TO_INT32_IN_MEM: 10834 case X86::FP64_TO_INT64_IN_MEM: 10835 case X86::FP80_TO_INT16_IN_MEM: 10836 case X86::FP80_TO_INT32_IN_MEM: 10837 case X86::FP80_TO_INT64_IN_MEM: { 10838 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 10839 DebugLoc DL = MI->getDebugLoc(); 10840 10841 // Change the floating point control register to use "round towards zero" 10842 // mode when truncating to an integer value. 10843 MachineFunction *F = BB->getParent(); 10844 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2, false); 10845 addFrameReference(BuildMI(*BB, MI, DL, 10846 TII->get(X86::FNSTCW16m)), CWFrameIdx); 10847 10848 // Load the old value of the high byte of the control word... 10849 unsigned OldCW = 10850 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 10851 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16rm), OldCW), 10852 CWFrameIdx); 10853 10854 // Set the high part to be round to zero... 10855 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mi)), CWFrameIdx) 10856 .addImm(0xC7F); 10857 10858 // Reload the modified control word now... 10859 addFrameReference(BuildMI(*BB, MI, DL, 10860 TII->get(X86::FLDCW16m)), CWFrameIdx); 10861 10862 // Restore the memory image of control word to original value 10863 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)), CWFrameIdx) 10864 .addReg(OldCW); 10865 10866 // Get the X86 opcode to use. 10867 unsigned Opc; 10868 switch (MI->getOpcode()) { 10869 default: llvm_unreachable("illegal opcode!"); 10870 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 10871 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 10872 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 10873 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 10874 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 10875 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 10876 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 10877 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 10878 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 10879 } 10880 10881 X86AddressMode AM; 10882 MachineOperand &Op = MI->getOperand(0); 10883 if (Op.isReg()) { 10884 AM.BaseType = X86AddressMode::RegBase; 10885 AM.Base.Reg = Op.getReg(); 10886 } else { 10887 AM.BaseType = X86AddressMode::FrameIndexBase; 10888 AM.Base.FrameIndex = Op.getIndex(); 10889 } 10890 Op = MI->getOperand(1); 10891 if (Op.isImm()) 10892 AM.Scale = Op.getImm(); 10893 Op = MI->getOperand(2); 10894 if (Op.isImm()) 10895 AM.IndexReg = Op.getImm(); 10896 Op = MI->getOperand(3); 10897 if (Op.isGlobal()) { 10898 AM.GV = Op.getGlobal(); 10899 } else { 10900 AM.Disp = Op.getImm(); 10901 } 10902 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM) 10903 .addReg(MI->getOperand(X86::AddrNumOperands).getReg()); 10904 10905 // Reload the original control word now. 10906 addFrameReference(BuildMI(*BB, MI, DL, 10907 TII->get(X86::FLDCW16m)), CWFrameIdx); 10908 10909 MI->eraseFromParent(); // The pseudo instruction is gone now. 10910 return BB; 10911 } 10912 // String/text processing lowering. 10913 case X86::PCMPISTRM128REG: 10914 case X86::VPCMPISTRM128REG: 10915 return EmitPCMP(MI, BB, 3, false /* in-mem */); 10916 case X86::PCMPISTRM128MEM: 10917 case X86::VPCMPISTRM128MEM: 10918 return EmitPCMP(MI, BB, 3, true /* in-mem */); 10919 case X86::PCMPESTRM128REG: 10920 case X86::VPCMPESTRM128REG: 10921 return EmitPCMP(MI, BB, 5, false /* in mem */); 10922 case X86::PCMPESTRM128MEM: 10923 case X86::VPCMPESTRM128MEM: 10924 return EmitPCMP(MI, BB, 5, true /* in mem */); 10925 10926 // Thread synchronization. 10927 case X86::MONITOR: 10928 return EmitMonitor(MI, BB); 10929 case X86::MWAIT: 10930 return EmitMwait(MI, BB); 10931 10932 // Atomic Lowering. 10933 case X86::ATOMAND32: 10934 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 10935 X86::AND32ri, X86::MOV32rm, 10936 X86::LCMPXCHG32, 10937 X86::NOT32r, X86::EAX, 10938 X86::GR32RegisterClass); 10939 case X86::ATOMOR32: 10940 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 10941 X86::OR32ri, X86::MOV32rm, 10942 X86::LCMPXCHG32, 10943 X86::NOT32r, X86::EAX, 10944 X86::GR32RegisterClass); 10945 case X86::ATOMXOR32: 10946 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 10947 X86::XOR32ri, X86::MOV32rm, 10948 X86::LCMPXCHG32, 10949 X86::NOT32r, X86::EAX, 10950 X86::GR32RegisterClass); 10951 case X86::ATOMNAND32: 10952 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 10953 X86::AND32ri, X86::MOV32rm, 10954 X86::LCMPXCHG32, 10955 X86::NOT32r, X86::EAX, 10956 X86::GR32RegisterClass, true); 10957 case X86::ATOMMIN32: 10958 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 10959 case X86::ATOMMAX32: 10960 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 10961 case X86::ATOMUMIN32: 10962 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 10963 case X86::ATOMUMAX32: 10964 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 10965 10966 case X86::ATOMAND16: 10967 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 10968 X86::AND16ri, X86::MOV16rm, 10969 X86::LCMPXCHG16, 10970 X86::NOT16r, X86::AX, 10971 X86::GR16RegisterClass); 10972 case X86::ATOMOR16: 10973 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 10974 X86::OR16ri, X86::MOV16rm, 10975 X86::LCMPXCHG16, 10976 X86::NOT16r, X86::AX, 10977 X86::GR16RegisterClass); 10978 case X86::ATOMXOR16: 10979 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 10980 X86::XOR16ri, X86::MOV16rm, 10981 X86::LCMPXCHG16, 10982 X86::NOT16r, X86::AX, 10983 X86::GR16RegisterClass); 10984 case X86::ATOMNAND16: 10985 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 10986 X86::AND16ri, X86::MOV16rm, 10987 X86::LCMPXCHG16, 10988 X86::NOT16r, X86::AX, 10989 X86::GR16RegisterClass, true); 10990 case X86::ATOMMIN16: 10991 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 10992 case X86::ATOMMAX16: 10993 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 10994 case X86::ATOMUMIN16: 10995 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 10996 case X86::ATOMUMAX16: 10997 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 10998 10999 case X86::ATOMAND8: 11000 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 11001 X86::AND8ri, X86::MOV8rm, 11002 X86::LCMPXCHG8, 11003 X86::NOT8r, X86::AL, 11004 X86::GR8RegisterClass); 11005 case X86::ATOMOR8: 11006 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 11007 X86::OR8ri, X86::MOV8rm, 11008 X86::LCMPXCHG8, 11009 X86::NOT8r, X86::AL, 11010 X86::GR8RegisterClass); 11011 case X86::ATOMXOR8: 11012 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 11013 X86::XOR8ri, X86::MOV8rm, 11014 X86::LCMPXCHG8, 11015 X86::NOT8r, X86::AL, 11016 X86::GR8RegisterClass); 11017 case X86::ATOMNAND8: 11018 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 11019 X86::AND8ri, X86::MOV8rm, 11020 X86::LCMPXCHG8, 11021 X86::NOT8r, X86::AL, 11022 X86::GR8RegisterClass, true); 11023 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 11024 // This group is for 64-bit host. 11025 case X86::ATOMAND64: 11026 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 11027 X86::AND64ri32, X86::MOV64rm, 11028 X86::LCMPXCHG64, 11029 X86::NOT64r, X86::RAX, 11030 X86::GR64RegisterClass); 11031 case X86::ATOMOR64: 11032 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, 11033 X86::OR64ri32, X86::MOV64rm, 11034 X86::LCMPXCHG64, 11035 X86::NOT64r, X86::RAX, 11036 X86::GR64RegisterClass); 11037 case X86::ATOMXOR64: 11038 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, 11039 X86::XOR64ri32, X86::MOV64rm, 11040 X86::LCMPXCHG64, 11041 X86::NOT64r, X86::RAX, 11042 X86::GR64RegisterClass); 11043 case X86::ATOMNAND64: 11044 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 11045 X86::AND64ri32, X86::MOV64rm, 11046 X86::LCMPXCHG64, 11047 X86::NOT64r, X86::RAX, 11048 X86::GR64RegisterClass, true); 11049 case X86::ATOMMIN64: 11050 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); 11051 case X86::ATOMMAX64: 11052 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); 11053 case X86::ATOMUMIN64: 11054 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); 11055 case X86::ATOMUMAX64: 11056 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); 11057 11058 // This group does 64-bit operations on a 32-bit host. 11059 case X86::ATOMAND6432: 11060 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11061 X86::AND32rr, X86::AND32rr, 11062 X86::AND32ri, X86::AND32ri, 11063 false); 11064 case X86::ATOMOR6432: 11065 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11066 X86::OR32rr, X86::OR32rr, 11067 X86::OR32ri, X86::OR32ri, 11068 false); 11069 case X86::ATOMXOR6432: 11070 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11071 X86::XOR32rr, X86::XOR32rr, 11072 X86::XOR32ri, X86::XOR32ri, 11073 false); 11074 case X86::ATOMNAND6432: 11075 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11076 X86::AND32rr, X86::AND32rr, 11077 X86::AND32ri, X86::AND32ri, 11078 true); 11079 case X86::ATOMADD6432: 11080 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11081 X86::ADD32rr, X86::ADC32rr, 11082 X86::ADD32ri, X86::ADC32ri, 11083 false); 11084 case X86::ATOMSUB6432: 11085 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11086 X86::SUB32rr, X86::SBB32rr, 11087 X86::SUB32ri, X86::SBB32ri, 11088 false); 11089 case X86::ATOMSWAP6432: 11090 return EmitAtomicBit6432WithCustomInserter(MI, BB, 11091 X86::MOV32rr, X86::MOV32rr, 11092 X86::MOV32ri, X86::MOV32ri, 11093 false); 11094 case X86::VASTART_SAVE_XMM_REGS: 11095 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB); 11096 11097 case X86::VAARG_64: 11098 return EmitVAARG64WithCustomInserter(MI, BB); 11099 } 11100} 11101 11102//===----------------------------------------------------------------------===// 11103// X86 Optimization Hooks 11104//===----------------------------------------------------------------------===// 11105 11106void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 11107 const APInt &Mask, 11108 APInt &KnownZero, 11109 APInt &KnownOne, 11110 const SelectionDAG &DAG, 11111 unsigned Depth) const { 11112 unsigned Opc = Op.getOpcode(); 11113 assert((Opc >= ISD::BUILTIN_OP_END || 11114 Opc == ISD::INTRINSIC_WO_CHAIN || 11115 Opc == ISD::INTRINSIC_W_CHAIN || 11116 Opc == ISD::INTRINSIC_VOID) && 11117 "Should use MaskedValueIsZero if you don't know whether Op" 11118 " is a target node!"); 11119 11120 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 11121 switch (Opc) { 11122 default: break; 11123 case X86ISD::ADD: 11124 case X86ISD::SUB: 11125 case X86ISD::ADC: 11126 case X86ISD::SBB: 11127 case X86ISD::SMUL: 11128 case X86ISD::UMUL: 11129 case X86ISD::INC: 11130 case X86ISD::DEC: 11131 case X86ISD::OR: 11132 case X86ISD::XOR: 11133 case X86ISD::AND: 11134 // These nodes' second result is a boolean. 11135 if (Op.getResNo() == 0) 11136 break; 11137 // Fallthrough 11138 case X86ISD::SETCC: 11139 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 11140 Mask.getBitWidth() - 1); 11141 break; 11142 } 11143} 11144 11145unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op, 11146 unsigned Depth) const { 11147 // SETCC_CARRY sets the dest to ~0 for true or 0 for false. 11148 if (Op.getOpcode() == X86ISD::SETCC_CARRY) 11149 return Op.getValueType().getScalarType().getSizeInBits(); 11150 11151 // Fallback case. 11152 return 1; 11153} 11154 11155/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 11156/// node is a GlobalAddress + offset. 11157bool X86TargetLowering::isGAPlusOffset(SDNode *N, 11158 const GlobalValue* &GA, 11159 int64_t &Offset) const { 11160 if (N->getOpcode() == X86ISD::Wrapper) { 11161 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 11162 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 11163 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset(); 11164 return true; 11165 } 11166 } 11167 return TargetLowering::isGAPlusOffset(N, GA, Offset); 11168} 11169 11170/// PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors. 11171static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG, 11172 TargetLowering::DAGCombinerInfo &DCI) { 11173 DebugLoc dl = N->getDebugLoc(); 11174 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 11175 SDValue V1 = SVOp->getOperand(0); 11176 SDValue V2 = SVOp->getOperand(1); 11177 EVT VT = SVOp->getValueType(0); 11178 11179 if (V1.getOpcode() == ISD::CONCAT_VECTORS && 11180 V2.getOpcode() == ISD::CONCAT_VECTORS) { 11181 // 11182 // 0,0,0,... 11183 // | 11184 // V UNDEF BUILD_VECTOR UNDEF 11185 // \ / \ / 11186 // CONCAT_VECTOR CONCAT_VECTOR 11187 // \ / 11188 // \ / 11189 // RESULT: V + zero extended 11190 // 11191 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR || 11192 V2.getOperand(1).getOpcode() != ISD::UNDEF || 11193 V1.getOperand(1).getOpcode() != ISD::UNDEF) 11194 return SDValue(); 11195 11196 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode())) 11197 return SDValue(); 11198 11199 // To match the shuffle mask, the first half of the mask should 11200 // be exactly the first vector, and all the rest a splat with the 11201 // first element of the second one. 11202 int NumElems = VT.getVectorNumElements(); 11203 for (int i = 0; i < NumElems/2; ++i) 11204 if (!isUndefOrEqual(SVOp->getMaskElt(i), i) || 11205 !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems)) 11206 return SDValue(); 11207 11208 // Emit a zeroed vector and insert the desired subvector on its 11209 // first half. 11210 SDValue Zeros = getZeroVector(VT, true /* HasSSE2 */, DAG, dl); 11211 SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 11212 DAG.getConstant(0, MVT::i32), DAG, dl); 11213 return DCI.CombineTo(N, InsV); 11214 } 11215 11216 return SDValue(); 11217} 11218 11219/// PerformShuffleCombine - Performs several different shuffle combines. 11220static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 11221 TargetLowering::DAGCombinerInfo &DCI) { 11222 DebugLoc dl = N->getDebugLoc(); 11223 EVT VT = N->getValueType(0); 11224 11225 // Don't create instructions with illegal types after legalize types has run. 11226 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11227 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType())) 11228 return SDValue(); 11229 11230 // Only handle pure VECTOR_SHUFFLE nodes. 11231 if (VT.getSizeInBits() == 256 && N->getOpcode() == ISD::VECTOR_SHUFFLE) 11232 return PerformShuffleCombine256(N, DAG, DCI); 11233 11234 // Only handle 128 wide vector from here on. 11235 if (VT.getSizeInBits() != 128) 11236 return SDValue(); 11237 11238 // Combine a vector_shuffle that is equal to build_vector load1, load2, load3, 11239 // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are 11240 // consecutive, non-overlapping, and in the right order. 11241 SmallVector<SDValue, 16> Elts; 11242 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) 11243 Elts.push_back(getShuffleScalarElt(N, i, DAG, 0)); 11244 11245 return EltsFromConsecutiveLoads(VT, Elts, dl, DAG); 11246} 11247 11248/// PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index 11249/// generation and convert it from being a bunch of shuffles and extracts 11250/// to a simple store and scalar loads to extract the elements. 11251static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, 11252 const TargetLowering &TLI) { 11253 SDValue InputVector = N->getOperand(0); 11254 11255 // Only operate on vectors of 4 elements, where the alternative shuffling 11256 // gets to be more expensive. 11257 if (InputVector.getValueType() != MVT::v4i32) 11258 return SDValue(); 11259 11260 // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a 11261 // single use which is a sign-extend or zero-extend, and all elements are 11262 // used. 11263 SmallVector<SDNode *, 4> Uses; 11264 unsigned ExtractedElements = 0; 11265 for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(), 11266 UE = InputVector.getNode()->use_end(); UI != UE; ++UI) { 11267 if (UI.getUse().getResNo() != InputVector.getResNo()) 11268 return SDValue(); 11269 11270 SDNode *Extract = *UI; 11271 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 11272 return SDValue(); 11273 11274 if (Extract->getValueType(0) != MVT::i32) 11275 return SDValue(); 11276 if (!Extract->hasOneUse()) 11277 return SDValue(); 11278 if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND && 11279 Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND) 11280 return SDValue(); 11281 if (!isa<ConstantSDNode>(Extract->getOperand(1))) 11282 return SDValue(); 11283 11284 // Record which element was extracted. 11285 ExtractedElements |= 11286 1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue(); 11287 11288 Uses.push_back(Extract); 11289 } 11290 11291 // If not all the elements were used, this may not be worthwhile. 11292 if (ExtractedElements != 15) 11293 return SDValue(); 11294 11295 // Ok, we've now decided to do the transformation. 11296 DebugLoc dl = InputVector.getDebugLoc(); 11297 11298 // Store the value to a temporary stack slot. 11299 SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType()); 11300 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr, 11301 MachinePointerInfo(), false, false, 0); 11302 11303 // Replace each use (extract) with a load of the appropriate element. 11304 for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(), 11305 UE = Uses.end(); UI != UE; ++UI) { 11306 SDNode *Extract = *UI; 11307 11308 // cOMpute the element's address. 11309 SDValue Idx = Extract->getOperand(1); 11310 unsigned EltSize = 11311 InputVector.getValueType().getVectorElementType().getSizeInBits()/8; 11312 uint64_t Offset = EltSize * cast<ConstantSDNode>(Idx)->getZExtValue(); 11313 SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy()); 11314 11315 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), 11316 StackPtr, OffsetVal); 11317 11318 // Load the scalar. 11319 SDValue LoadScalar = DAG.getLoad(Extract->getValueType(0), dl, Ch, 11320 ScalarAddr, MachinePointerInfo(), 11321 false, false, 0); 11322 11323 // Replace the exact with the load. 11324 DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), LoadScalar); 11325 } 11326 11327 // The replacement was made in place; don't return anything. 11328 return SDValue(); 11329} 11330 11331/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 11332static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 11333 const X86Subtarget *Subtarget) { 11334 DebugLoc DL = N->getDebugLoc(); 11335 SDValue Cond = N->getOperand(0); 11336 // Get the LHS/RHS of the select. 11337 SDValue LHS = N->getOperand(1); 11338 SDValue RHS = N->getOperand(2); 11339 11340 // If we have SSE[12] support, try to form min/max nodes. SSE min/max 11341 // instructions match the semantics of the common C idiom x<y?x:y but not 11342 // x<=y?x:y, because of how they handle negative zero (which can be 11343 // ignored in unsafe-math mode). 11344 if (Subtarget->hasSSE2() && 11345 (LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64) && 11346 Cond.getOpcode() == ISD::SETCC) { 11347 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 11348 11349 unsigned Opcode = 0; 11350 // Check for x CC y ? x : y. 11351 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) && 11352 DAG.isEqualTo(RHS, Cond.getOperand(1))) { 11353 switch (CC) { 11354 default: break; 11355 case ISD::SETULT: 11356 // Converting this to a min would handle NaNs incorrectly, and swapping 11357 // the operands would cause it to handle comparisons between positive 11358 // and negative zero incorrectly. 11359 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { 11360 if (!UnsafeFPMath && 11361 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) 11362 break; 11363 std::swap(LHS, RHS); 11364 } 11365 Opcode = X86ISD::FMIN; 11366 break; 11367 case ISD::SETOLE: 11368 // Converting this to a min would handle comparisons between positive 11369 // and negative zero incorrectly. 11370 if (!UnsafeFPMath && 11371 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) 11372 break; 11373 Opcode = X86ISD::FMIN; 11374 break; 11375 case ISD::SETULE: 11376 // Converting this to a min would handle both negative zeros and NaNs 11377 // incorrectly, but we can swap the operands to fix both. 11378 std::swap(LHS, RHS); 11379 case ISD::SETOLT: 11380 case ISD::SETLT: 11381 case ISD::SETLE: 11382 Opcode = X86ISD::FMIN; 11383 break; 11384 11385 case ISD::SETOGE: 11386 // Converting this to a max would handle comparisons between positive 11387 // and negative zero incorrectly. 11388 if (!UnsafeFPMath && 11389 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(LHS)) 11390 break; 11391 Opcode = X86ISD::FMAX; 11392 break; 11393 case ISD::SETUGT: 11394 // Converting this to a max would handle NaNs incorrectly, and swapping 11395 // the operands would cause it to handle comparisons between positive 11396 // and negative zero incorrectly. 11397 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { 11398 if (!UnsafeFPMath && 11399 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) 11400 break; 11401 std::swap(LHS, RHS); 11402 } 11403 Opcode = X86ISD::FMAX; 11404 break; 11405 case ISD::SETUGE: 11406 // Converting this to a max would handle both negative zeros and NaNs 11407 // incorrectly, but we can swap the operands to fix both. 11408 std::swap(LHS, RHS); 11409 case ISD::SETOGT: 11410 case ISD::SETGT: 11411 case ISD::SETGE: 11412 Opcode = X86ISD::FMAX; 11413 break; 11414 } 11415 // Check for x CC y ? y : x -- a min/max with reversed arms. 11416 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) && 11417 DAG.isEqualTo(RHS, Cond.getOperand(0))) { 11418 switch (CC) { 11419 default: break; 11420 case ISD::SETOGE: 11421 // Converting this to a min would handle comparisons between positive 11422 // and negative zero incorrectly, and swapping the operands would 11423 // cause it to handle NaNs incorrectly. 11424 if (!UnsafeFPMath && 11425 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) { 11426 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 11427 break; 11428 std::swap(LHS, RHS); 11429 } 11430 Opcode = X86ISD::FMIN; 11431 break; 11432 case ISD::SETUGT: 11433 // Converting this to a min would handle NaNs incorrectly. 11434 if (!UnsafeFPMath && 11435 (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))) 11436 break; 11437 Opcode = X86ISD::FMIN; 11438 break; 11439 case ISD::SETUGE: 11440 // Converting this to a min would handle both negative zeros and NaNs 11441 // incorrectly, but we can swap the operands to fix both. 11442 std::swap(LHS, RHS); 11443 case ISD::SETOGT: 11444 case ISD::SETGT: 11445 case ISD::SETGE: 11446 Opcode = X86ISD::FMIN; 11447 break; 11448 11449 case ISD::SETULT: 11450 // Converting this to a max would handle NaNs incorrectly. 11451 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 11452 break; 11453 Opcode = X86ISD::FMAX; 11454 break; 11455 case ISD::SETOLE: 11456 // Converting this to a max would handle comparisons between positive 11457 // and negative zero incorrectly, and swapping the operands would 11458 // cause it to handle NaNs incorrectly. 11459 if (!UnsafeFPMath && 11460 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) { 11461 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 11462 break; 11463 std::swap(LHS, RHS); 11464 } 11465 Opcode = X86ISD::FMAX; 11466 break; 11467 case ISD::SETULE: 11468 // Converting this to a max would handle both negative zeros and NaNs 11469 // incorrectly, but we can swap the operands to fix both. 11470 std::swap(LHS, RHS); 11471 case ISD::SETOLT: 11472 case ISD::SETLT: 11473 case ISD::SETLE: 11474 Opcode = X86ISD::FMAX; 11475 break; 11476 } 11477 } 11478 11479 if (Opcode) 11480 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); 11481 } 11482 11483 // If this is a select between two integer constants, try to do some 11484 // optimizations. 11485 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) { 11486 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS)) 11487 // Don't do this for crazy integer types. 11488 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) { 11489 // If this is efficiently invertible, canonicalize the LHSC/RHSC values 11490 // so that TrueC (the true value) is larger than FalseC. 11491 bool NeedsCondInvert = false; 11492 11493 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) && 11494 // Efficiently invertible. 11495 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible. 11496 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible. 11497 isa<ConstantSDNode>(Cond.getOperand(1))))) { 11498 NeedsCondInvert = true; 11499 std::swap(TrueC, FalseC); 11500 } 11501 11502 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0. 11503 if (FalseC->getAPIntValue() == 0 && 11504 TrueC->getAPIntValue().isPowerOf2()) { 11505 if (NeedsCondInvert) // Invert the condition if needed. 11506 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 11507 DAG.getConstant(1, Cond.getValueType())); 11508 11509 // Zero extend the condition if needed. 11510 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond); 11511 11512 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 11513 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond, 11514 DAG.getConstant(ShAmt, MVT::i8)); 11515 } 11516 11517 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. 11518 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 11519 if (NeedsCondInvert) // Invert the condition if needed. 11520 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 11521 DAG.getConstant(1, Cond.getValueType())); 11522 11523 // Zero extend the condition if needed. 11524 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 11525 FalseC->getValueType(0), Cond); 11526 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 11527 SDValue(FalseC, 0)); 11528 } 11529 11530 // Optimize cases that will turn into an LEA instruction. This requires 11531 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 11532 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 11533 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 11534 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 11535 11536 bool isFastMultiplier = false; 11537 if (Diff < 10) { 11538 switch ((unsigned char)Diff) { 11539 default: break; 11540 case 1: // result = add base, cond 11541 case 2: // result = lea base( , cond*2) 11542 case 3: // result = lea base(cond, cond*2) 11543 case 4: // result = lea base( , cond*4) 11544 case 5: // result = lea base(cond, cond*4) 11545 case 8: // result = lea base( , cond*8) 11546 case 9: // result = lea base(cond, cond*8) 11547 isFastMultiplier = true; 11548 break; 11549 } 11550 } 11551 11552 if (isFastMultiplier) { 11553 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 11554 if (NeedsCondInvert) // Invert the condition if needed. 11555 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 11556 DAG.getConstant(1, Cond.getValueType())); 11557 11558 // Zero extend the condition if needed. 11559 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 11560 Cond); 11561 // Scale the condition by the difference. 11562 if (Diff != 1) 11563 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 11564 DAG.getConstant(Diff, Cond.getValueType())); 11565 11566 // Add the base if non-zero. 11567 if (FalseC->getAPIntValue() != 0) 11568 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 11569 SDValue(FalseC, 0)); 11570 return Cond; 11571 } 11572 } 11573 } 11574 } 11575 11576 return SDValue(); 11577} 11578 11579/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL] 11580static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG, 11581 TargetLowering::DAGCombinerInfo &DCI) { 11582 DebugLoc DL = N->getDebugLoc(); 11583 11584 // If the flag operand isn't dead, don't touch this CMOV. 11585 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty()) 11586 return SDValue(); 11587 11588 SDValue FalseOp = N->getOperand(0); 11589 SDValue TrueOp = N->getOperand(1); 11590 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2); 11591 SDValue Cond = N->getOperand(3); 11592 if (CC == X86::COND_E || CC == X86::COND_NE) { 11593 switch (Cond.getOpcode()) { 11594 default: break; 11595 case X86ISD::BSR: 11596 case X86ISD::BSF: 11597 // If operand of BSR / BSF are proven never zero, then ZF cannot be set. 11598 if (DAG.isKnownNeverZero(Cond.getOperand(0))) 11599 return (CC == X86::COND_E) ? FalseOp : TrueOp; 11600 } 11601 } 11602 11603 // If this is a select between two integer constants, try to do some 11604 // optimizations. Note that the operands are ordered the opposite of SELECT 11605 // operands. 11606 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) { 11607 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) { 11608 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is 11609 // larger than FalseC (the false value). 11610 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) { 11611 CC = X86::GetOppositeBranchCondition(CC); 11612 std::swap(TrueC, FalseC); 11613 } 11614 11615 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0. 11616 // This is efficient for any integer data type (including i8/i16) and 11617 // shift amount. 11618 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) { 11619 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 11620 DAG.getConstant(CC, MVT::i8), Cond); 11621 11622 // Zero extend the condition if needed. 11623 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); 11624 11625 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 11626 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, 11627 DAG.getConstant(ShAmt, MVT::i8)); 11628 if (N->getNumValues() == 2) // Dead flag value? 11629 return DCI.CombineTo(N, Cond, SDValue()); 11630 return Cond; 11631 } 11632 11633 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient 11634 // for any integer data type, including i8/i16. 11635 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 11636 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 11637 DAG.getConstant(CC, MVT::i8), Cond); 11638 11639 // Zero extend the condition if needed. 11640 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 11641 FalseC->getValueType(0), Cond); 11642 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 11643 SDValue(FalseC, 0)); 11644 11645 if (N->getNumValues() == 2) // Dead flag value? 11646 return DCI.CombineTo(N, Cond, SDValue()); 11647 return Cond; 11648 } 11649 11650 // Optimize cases that will turn into an LEA instruction. This requires 11651 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 11652 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 11653 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 11654 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 11655 11656 bool isFastMultiplier = false; 11657 if (Diff < 10) { 11658 switch ((unsigned char)Diff) { 11659 default: break; 11660 case 1: // result = add base, cond 11661 case 2: // result = lea base( , cond*2) 11662 case 3: // result = lea base(cond, cond*2) 11663 case 4: // result = lea base( , cond*4) 11664 case 5: // result = lea base(cond, cond*4) 11665 case 8: // result = lea base( , cond*8) 11666 case 9: // result = lea base(cond, cond*8) 11667 isFastMultiplier = true; 11668 break; 11669 } 11670 } 11671 11672 if (isFastMultiplier) { 11673 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 11674 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 11675 DAG.getConstant(CC, MVT::i8), Cond); 11676 // Zero extend the condition if needed. 11677 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 11678 Cond); 11679 // Scale the condition by the difference. 11680 if (Diff != 1) 11681 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 11682 DAG.getConstant(Diff, Cond.getValueType())); 11683 11684 // Add the base if non-zero. 11685 if (FalseC->getAPIntValue() != 0) 11686 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 11687 SDValue(FalseC, 0)); 11688 if (N->getNumValues() == 2) // Dead flag value? 11689 return DCI.CombineTo(N, Cond, SDValue()); 11690 return Cond; 11691 } 11692 } 11693 } 11694 } 11695 return SDValue(); 11696} 11697 11698 11699/// PerformMulCombine - Optimize a single multiply with constant into two 11700/// in order to implement it with two cheaper instructions, e.g. 11701/// LEA + SHL, LEA + LEA. 11702static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG, 11703 TargetLowering::DAGCombinerInfo &DCI) { 11704 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 11705 return SDValue(); 11706 11707 EVT VT = N->getValueType(0); 11708 if (VT != MVT::i64) 11709 return SDValue(); 11710 11711 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 11712 if (!C) 11713 return SDValue(); 11714 uint64_t MulAmt = C->getZExtValue(); 11715 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9) 11716 return SDValue(); 11717 11718 uint64_t MulAmt1 = 0; 11719 uint64_t MulAmt2 = 0; 11720 if ((MulAmt % 9) == 0) { 11721 MulAmt1 = 9; 11722 MulAmt2 = MulAmt / 9; 11723 } else if ((MulAmt % 5) == 0) { 11724 MulAmt1 = 5; 11725 MulAmt2 = MulAmt / 5; 11726 } else if ((MulAmt % 3) == 0) { 11727 MulAmt1 = 3; 11728 MulAmt2 = MulAmt / 3; 11729 } 11730 if (MulAmt2 && 11731 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){ 11732 DebugLoc DL = N->getDebugLoc(); 11733 11734 if (isPowerOf2_64(MulAmt2) && 11735 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD)) 11736 // If second multiplifer is pow2, issue it first. We want the multiply by 11737 // 3, 5, or 9 to be folded into the addressing mode unless the lone use 11738 // is an add. 11739 std::swap(MulAmt1, MulAmt2); 11740 11741 SDValue NewMul; 11742 if (isPowerOf2_64(MulAmt1)) 11743 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 11744 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); 11745 else 11746 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), 11747 DAG.getConstant(MulAmt1, VT)); 11748 11749 if (isPowerOf2_64(MulAmt2)) 11750 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, 11751 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); 11752 else 11753 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, 11754 DAG.getConstant(MulAmt2, VT)); 11755 11756 // Do not add new nodes to DAG combiner worklist. 11757 DCI.CombineTo(N, NewMul, false); 11758 } 11759 return SDValue(); 11760} 11761 11762static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) { 11763 SDValue N0 = N->getOperand(0); 11764 SDValue N1 = N->getOperand(1); 11765 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 11766 EVT VT = N0.getValueType(); 11767 11768 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2)) 11769 // since the result of setcc_c is all zero's or all ones. 11770 if (N1C && N0.getOpcode() == ISD::AND && 11771 N0.getOperand(1).getOpcode() == ISD::Constant) { 11772 SDValue N00 = N0.getOperand(0); 11773 if (N00.getOpcode() == X86ISD::SETCC_CARRY || 11774 ((N00.getOpcode() == ISD::ANY_EXTEND || 11775 N00.getOpcode() == ISD::ZERO_EXTEND) && 11776 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) { 11777 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 11778 APInt ShAmt = N1C->getAPIntValue(); 11779 Mask = Mask.shl(ShAmt); 11780 if (Mask != 0) 11781 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, 11782 N00, DAG.getConstant(Mask, VT)); 11783 } 11784 } 11785 11786 return SDValue(); 11787} 11788 11789/// PerformShiftCombine - Transforms vector shift nodes to use vector shifts 11790/// when possible. 11791static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG, 11792 const X86Subtarget *Subtarget) { 11793 EVT VT = N->getValueType(0); 11794 if (!VT.isVector() && VT.isInteger() && 11795 N->getOpcode() == ISD::SHL) 11796 return PerformSHLCombine(N, DAG); 11797 11798 // On X86 with SSE2 support, we can transform this to a vector shift if 11799 // all elements are shifted by the same amount. We can't do this in legalize 11800 // because the a constant vector is typically transformed to a constant pool 11801 // so we have no knowledge of the shift amount. 11802 if (!Subtarget->hasSSE2()) 11803 return SDValue(); 11804 11805 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16) 11806 return SDValue(); 11807 11808 SDValue ShAmtOp = N->getOperand(1); 11809 EVT EltVT = VT.getVectorElementType(); 11810 DebugLoc DL = N->getDebugLoc(); 11811 SDValue BaseShAmt = SDValue(); 11812 if (ShAmtOp.getOpcode() == ISD::BUILD_VECTOR) { 11813 unsigned NumElts = VT.getVectorNumElements(); 11814 unsigned i = 0; 11815 for (; i != NumElts; ++i) { 11816 SDValue Arg = ShAmtOp.getOperand(i); 11817 if (Arg.getOpcode() == ISD::UNDEF) continue; 11818 BaseShAmt = Arg; 11819 break; 11820 } 11821 for (; i != NumElts; ++i) { 11822 SDValue Arg = ShAmtOp.getOperand(i); 11823 if (Arg.getOpcode() == ISD::UNDEF) continue; 11824 if (Arg != BaseShAmt) { 11825 return SDValue(); 11826 } 11827 } 11828 } else if (ShAmtOp.getOpcode() == ISD::VECTOR_SHUFFLE && 11829 cast<ShuffleVectorSDNode>(ShAmtOp)->isSplat()) { 11830 SDValue InVec = ShAmtOp.getOperand(0); 11831 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 11832 unsigned NumElts = InVec.getValueType().getVectorNumElements(); 11833 unsigned i = 0; 11834 for (; i != NumElts; ++i) { 11835 SDValue Arg = InVec.getOperand(i); 11836 if (Arg.getOpcode() == ISD::UNDEF) continue; 11837 BaseShAmt = Arg; 11838 break; 11839 } 11840 } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) { 11841 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(InVec.getOperand(2))) { 11842 unsigned SplatIdx= cast<ShuffleVectorSDNode>(ShAmtOp)->getSplatIndex(); 11843 if (C->getZExtValue() == SplatIdx) 11844 BaseShAmt = InVec.getOperand(1); 11845 } 11846 } 11847 if (BaseShAmt.getNode() == 0) 11848 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp, 11849 DAG.getIntPtrConstant(0)); 11850 } else 11851 return SDValue(); 11852 11853 // The shift amount is an i32. 11854 if (EltVT.bitsGT(MVT::i32)) 11855 BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt); 11856 else if (EltVT.bitsLT(MVT::i32)) 11857 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseShAmt); 11858 11859 // The shift amount is identical so we can do a vector shift. 11860 SDValue ValOp = N->getOperand(0); 11861 switch (N->getOpcode()) { 11862 default: 11863 llvm_unreachable("Unknown shift opcode!"); 11864 break; 11865 case ISD::SHL: 11866 if (VT == MVT::v2i64) 11867 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11868 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 11869 ValOp, BaseShAmt); 11870 if (VT == MVT::v4i32) 11871 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11872 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), 11873 ValOp, BaseShAmt); 11874 if (VT == MVT::v8i16) 11875 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11876 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), 11877 ValOp, BaseShAmt); 11878 break; 11879 case ISD::SRA: 11880 if (VT == MVT::v4i32) 11881 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11882 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32), 11883 ValOp, BaseShAmt); 11884 if (VT == MVT::v8i16) 11885 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11886 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32), 11887 ValOp, BaseShAmt); 11888 break; 11889 case ISD::SRL: 11890 if (VT == MVT::v2i64) 11891 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11892 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 11893 ValOp, BaseShAmt); 11894 if (VT == MVT::v4i32) 11895 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11896 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32), 11897 ValOp, BaseShAmt); 11898 if (VT == MVT::v8i16) 11899 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 11900 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32), 11901 ValOp, BaseShAmt); 11902 break; 11903 } 11904 return SDValue(); 11905} 11906 11907 11908// CMPEQCombine - Recognize the distinctive (AND (setcc ...) (setcc ..)) 11909// where both setccs reference the same FP CMP, and rewrite for CMPEQSS 11910// and friends. Likewise for OR -> CMPNEQSS. 11911static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG, 11912 TargetLowering::DAGCombinerInfo &DCI, 11913 const X86Subtarget *Subtarget) { 11914 unsigned opcode; 11915 11916 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but 11917 // we're requiring SSE2 for both. 11918 if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) { 11919 SDValue N0 = N->getOperand(0); 11920 SDValue N1 = N->getOperand(1); 11921 SDValue CMP0 = N0->getOperand(1); 11922 SDValue CMP1 = N1->getOperand(1); 11923 DebugLoc DL = N->getDebugLoc(); 11924 11925 // The SETCCs should both refer to the same CMP. 11926 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1) 11927 return SDValue(); 11928 11929 SDValue CMP00 = CMP0->getOperand(0); 11930 SDValue CMP01 = CMP0->getOperand(1); 11931 EVT VT = CMP00.getValueType(); 11932 11933 if (VT == MVT::f32 || VT == MVT::f64) { 11934 bool ExpectingFlags = false; 11935 // Check for any users that want flags: 11936 for (SDNode::use_iterator UI = N->use_begin(), 11937 UE = N->use_end(); 11938 !ExpectingFlags && UI != UE; ++UI) 11939 switch (UI->getOpcode()) { 11940 default: 11941 case ISD::BR_CC: 11942 case ISD::BRCOND: 11943 case ISD::SELECT: 11944 ExpectingFlags = true; 11945 break; 11946 case ISD::CopyToReg: 11947 case ISD::SIGN_EXTEND: 11948 case ISD::ZERO_EXTEND: 11949 case ISD::ANY_EXTEND: 11950 break; 11951 } 11952 11953 if (!ExpectingFlags) { 11954 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0); 11955 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0); 11956 11957 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) { 11958 X86::CondCode tmp = cc0; 11959 cc0 = cc1; 11960 cc1 = tmp; 11961 } 11962 11963 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) || 11964 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) { 11965 bool is64BitFP = (CMP00.getValueType() == MVT::f64); 11966 X86ISD::NodeType NTOperator = is64BitFP ? 11967 X86ISD::FSETCCsd : X86ISD::FSETCCss; 11968 // FIXME: need symbolic constants for these magic numbers. 11969 // See X86ATTInstPrinter.cpp:printSSECC(). 11970 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4; 11971 SDValue OnesOrZeroesF = DAG.getNode(NTOperator, DL, MVT::f32, CMP00, CMP01, 11972 DAG.getConstant(x86cc, MVT::i8)); 11973 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, MVT::i32, 11974 OnesOrZeroesF); 11975 SDValue ANDed = DAG.getNode(ISD::AND, DL, MVT::i32, OnesOrZeroesI, 11976 DAG.getConstant(1, MVT::i32)); 11977 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); 11978 return OneBitOfTruth; 11979 } 11980 } 11981 } 11982 } 11983 return SDValue(); 11984} 11985 11986static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, 11987 TargetLowering::DAGCombinerInfo &DCI, 11988 const X86Subtarget *Subtarget) { 11989 if (DCI.isBeforeLegalizeOps()) 11990 return SDValue(); 11991 11992 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); 11993 if (R.getNode()) 11994 return R; 11995 11996 // Want to form ANDNP nodes: 11997 // 1) In the hopes of then easily combining them with OR and AND nodes 11998 // to form PBLEND/PSIGN. 11999 // 2) To match ANDN packed intrinsics 12000 EVT VT = N->getValueType(0); 12001 if (VT != MVT::v2i64 && VT != MVT::v4i64) 12002 return SDValue(); 12003 12004 SDValue N0 = N->getOperand(0); 12005 SDValue N1 = N->getOperand(1); 12006 DebugLoc DL = N->getDebugLoc(); 12007 12008 // Check LHS for vnot 12009 if (N0.getOpcode() == ISD::XOR && 12010 ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode())) 12011 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1); 12012 12013 // Check RHS for vnot 12014 if (N1.getOpcode() == ISD::XOR && 12015 ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode())) 12016 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0); 12017 12018 return SDValue(); 12019} 12020 12021static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, 12022 TargetLowering::DAGCombinerInfo &DCI, 12023 const X86Subtarget *Subtarget) { 12024 if (DCI.isBeforeLegalizeOps()) 12025 return SDValue(); 12026 12027 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); 12028 if (R.getNode()) 12029 return R; 12030 12031 EVT VT = N->getValueType(0); 12032 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64 && VT != MVT::v2i64) 12033 return SDValue(); 12034 12035 SDValue N0 = N->getOperand(0); 12036 SDValue N1 = N->getOperand(1); 12037 12038 // look for psign/blend 12039 if (Subtarget->hasSSSE3()) { 12040 if (VT == MVT::v2i64) { 12041 // Canonicalize pandn to RHS 12042 if (N0.getOpcode() == X86ISD::ANDNP) 12043 std::swap(N0, N1); 12044 // or (and (m, x), (pandn m, y)) 12045 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) { 12046 SDValue Mask = N1.getOperand(0); 12047 SDValue X = N1.getOperand(1); 12048 SDValue Y; 12049 if (N0.getOperand(0) == Mask) 12050 Y = N0.getOperand(1); 12051 if (N0.getOperand(1) == Mask) 12052 Y = N0.getOperand(0); 12053 12054 // Check to see if the mask appeared in both the AND and ANDNP and 12055 if (!Y.getNode()) 12056 return SDValue(); 12057 12058 // Validate that X, Y, and Mask are BIT_CONVERTS, and see through them. 12059 if (Mask.getOpcode() != ISD::BITCAST || 12060 X.getOpcode() != ISD::BITCAST || 12061 Y.getOpcode() != ISD::BITCAST) 12062 return SDValue(); 12063 12064 // Look through mask bitcast. 12065 Mask = Mask.getOperand(0); 12066 EVT MaskVT = Mask.getValueType(); 12067 12068 // Validate that the Mask operand is a vector sra node. The sra node 12069 // will be an intrinsic. 12070 if (Mask.getOpcode() != ISD::INTRINSIC_WO_CHAIN) 12071 return SDValue(); 12072 12073 // FIXME: what to do for bytes, since there is a psignb/pblendvb, but 12074 // there is no psrai.b 12075 switch (cast<ConstantSDNode>(Mask.getOperand(0))->getZExtValue()) { 12076 case Intrinsic::x86_sse2_psrai_w: 12077 case Intrinsic::x86_sse2_psrai_d: 12078 break; 12079 default: return SDValue(); 12080 } 12081 12082 // Check that the SRA is all signbits. 12083 SDValue SraC = Mask.getOperand(2); 12084 unsigned SraAmt = cast<ConstantSDNode>(SraC)->getZExtValue(); 12085 unsigned EltBits = MaskVT.getVectorElementType().getSizeInBits(); 12086 if ((SraAmt + 1) != EltBits) 12087 return SDValue(); 12088 12089 DebugLoc DL = N->getDebugLoc(); 12090 12091 // Now we know we at least have a plendvb with the mask val. See if 12092 // we can form a psignb/w/d. 12093 // psign = x.type == y.type == mask.type && y = sub(0, x); 12094 X = X.getOperand(0); 12095 Y = Y.getOperand(0); 12096 if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X && 12097 ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) && 12098 X.getValueType() == MaskVT && X.getValueType() == Y.getValueType()){ 12099 unsigned Opc = 0; 12100 switch (EltBits) { 12101 case 8: Opc = X86ISD::PSIGNB; break; 12102 case 16: Opc = X86ISD::PSIGNW; break; 12103 case 32: Opc = X86ISD::PSIGND; break; 12104 default: break; 12105 } 12106 if (Opc) { 12107 SDValue Sign = DAG.getNode(Opc, DL, MaskVT, X, Mask.getOperand(1)); 12108 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Sign); 12109 } 12110 } 12111 // PBLENDVB only available on SSE 4.1 12112 if (!Subtarget->hasSSE41()) 12113 return SDValue(); 12114 12115 X = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, X); 12116 Y = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Y); 12117 Mask = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Mask); 12118 Mask = DAG.getNode(X86ISD::PBLENDVB, DL, MVT::v16i8, X, Y, Mask); 12119 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Mask); 12120 } 12121 } 12122 } 12123 12124 // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c) 12125 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) 12126 std::swap(N0, N1); 12127 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) 12128 return SDValue(); 12129 if (!N0.hasOneUse() || !N1.hasOneUse()) 12130 return SDValue(); 12131 12132 SDValue ShAmt0 = N0.getOperand(1); 12133 if (ShAmt0.getValueType() != MVT::i8) 12134 return SDValue(); 12135 SDValue ShAmt1 = N1.getOperand(1); 12136 if (ShAmt1.getValueType() != MVT::i8) 12137 return SDValue(); 12138 if (ShAmt0.getOpcode() == ISD::TRUNCATE) 12139 ShAmt0 = ShAmt0.getOperand(0); 12140 if (ShAmt1.getOpcode() == ISD::TRUNCATE) 12141 ShAmt1 = ShAmt1.getOperand(0); 12142 12143 DebugLoc DL = N->getDebugLoc(); 12144 unsigned Opc = X86ISD::SHLD; 12145 SDValue Op0 = N0.getOperand(0); 12146 SDValue Op1 = N1.getOperand(0); 12147 if (ShAmt0.getOpcode() == ISD::SUB) { 12148 Opc = X86ISD::SHRD; 12149 std::swap(Op0, Op1); 12150 std::swap(ShAmt0, ShAmt1); 12151 } 12152 12153 unsigned Bits = VT.getSizeInBits(); 12154 if (ShAmt1.getOpcode() == ISD::SUB) { 12155 SDValue Sum = ShAmt1.getOperand(0); 12156 if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) { 12157 SDValue ShAmt1Op1 = ShAmt1.getOperand(1); 12158 if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE) 12159 ShAmt1Op1 = ShAmt1Op1.getOperand(0); 12160 if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0) 12161 return DAG.getNode(Opc, DL, VT, 12162 Op0, Op1, 12163 DAG.getNode(ISD::TRUNCATE, DL, 12164 MVT::i8, ShAmt0)); 12165 } 12166 } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) { 12167 ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0); 12168 if (ShAmt0C && 12169 ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits) 12170 return DAG.getNode(Opc, DL, VT, 12171 N0.getOperand(0), N1.getOperand(0), 12172 DAG.getNode(ISD::TRUNCATE, DL, 12173 MVT::i8, ShAmt0)); 12174 } 12175 12176 return SDValue(); 12177} 12178 12179/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 12180static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 12181 const X86Subtarget *Subtarget) { 12182 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 12183 // the FP state in cases where an emms may be missing. 12184 // A preferable solution to the general problem is to figure out the right 12185 // places to insert EMMS. This qualifies as a quick hack. 12186 12187 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode. 12188 StoreSDNode *St = cast<StoreSDNode>(N); 12189 EVT VT = St->getValue().getValueType(); 12190 if (VT.getSizeInBits() != 64) 12191 return SDValue(); 12192 12193 const Function *F = DAG.getMachineFunction().getFunction(); 12194 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat); 12195 bool F64IsLegal = !UseSoftFloat && !NoImplicitFloatOps 12196 && Subtarget->hasSSE2(); 12197 if ((VT.isVector() || 12198 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && 12199 isa<LoadSDNode>(St->getValue()) && 12200 !cast<LoadSDNode>(St->getValue())->isVolatile() && 12201 St->getChain().hasOneUse() && !St->isVolatile()) { 12202 SDNode* LdVal = St->getValue().getNode(); 12203 LoadSDNode *Ld = 0; 12204 int TokenFactorIndex = -1; 12205 SmallVector<SDValue, 8> Ops; 12206 SDNode* ChainVal = St->getChain().getNode(); 12207 // Must be a store of a load. We currently handle two cases: the load 12208 // is a direct child, and it's under an intervening TokenFactor. It is 12209 // possible to dig deeper under nested TokenFactors. 12210 if (ChainVal == LdVal) 12211 Ld = cast<LoadSDNode>(St->getChain()); 12212 else if (St->getValue().hasOneUse() && 12213 ChainVal->getOpcode() == ISD::TokenFactor) { 12214 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 12215 if (ChainVal->getOperand(i).getNode() == LdVal) { 12216 TokenFactorIndex = i; 12217 Ld = cast<LoadSDNode>(St->getValue()); 12218 } else 12219 Ops.push_back(ChainVal->getOperand(i)); 12220 } 12221 } 12222 12223 if (!Ld || !ISD::isNormalLoad(Ld)) 12224 return SDValue(); 12225 12226 // If this is not the MMX case, i.e. we are just turning i64 load/store 12227 // into f64 load/store, avoid the transformation if there are multiple 12228 // uses of the loaded value. 12229 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0)) 12230 return SDValue(); 12231 12232 DebugLoc LdDL = Ld->getDebugLoc(); 12233 DebugLoc StDL = N->getDebugLoc(); 12234 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 12235 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store 12236 // pair instead. 12237 if (Subtarget->is64Bit() || F64IsLegal) { 12238 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; 12239 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(), 12240 Ld->getPointerInfo(), Ld->isVolatile(), 12241 Ld->isNonTemporal(), Ld->getAlignment()); 12242 SDValue NewChain = NewLd.getValue(1); 12243 if (TokenFactorIndex != -1) { 12244 Ops.push_back(NewChain); 12245 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 12246 Ops.size()); 12247 } 12248 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(), 12249 St->getPointerInfo(), 12250 St->isVolatile(), St->isNonTemporal(), 12251 St->getAlignment()); 12252 } 12253 12254 // Otherwise, lower to two pairs of 32-bit loads / stores. 12255 SDValue LoAddr = Ld->getBasePtr(); 12256 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, 12257 DAG.getConstant(4, MVT::i32)); 12258 12259 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, 12260 Ld->getPointerInfo(), 12261 Ld->isVolatile(), Ld->isNonTemporal(), 12262 Ld->getAlignment()); 12263 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, 12264 Ld->getPointerInfo().getWithOffset(4), 12265 Ld->isVolatile(), Ld->isNonTemporal(), 12266 MinAlign(Ld->getAlignment(), 4)); 12267 12268 SDValue NewChain = LoLd.getValue(1); 12269 if (TokenFactorIndex != -1) { 12270 Ops.push_back(LoLd); 12271 Ops.push_back(HiLd); 12272 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 12273 Ops.size()); 12274 } 12275 12276 LoAddr = St->getBasePtr(); 12277 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, 12278 DAG.getConstant(4, MVT::i32)); 12279 12280 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr, 12281 St->getPointerInfo(), 12282 St->isVolatile(), St->isNonTemporal(), 12283 St->getAlignment()); 12284 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr, 12285 St->getPointerInfo().getWithOffset(4), 12286 St->isVolatile(), 12287 St->isNonTemporal(), 12288 MinAlign(St->getAlignment(), 4)); 12289 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); 12290 } 12291 return SDValue(); 12292} 12293 12294/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 12295/// X86ISD::FXOR nodes. 12296static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 12297 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 12298 // F[X]OR(0.0, x) -> x 12299 // F[X]OR(x, 0.0) -> x 12300 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 12301 if (C->getValueAPF().isPosZero()) 12302 return N->getOperand(1); 12303 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 12304 if (C->getValueAPF().isPosZero()) 12305 return N->getOperand(0); 12306 return SDValue(); 12307} 12308 12309/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 12310static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 12311 // FAND(0.0, x) -> 0.0 12312 // FAND(x, 0.0) -> 0.0 12313 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 12314 if (C->getValueAPF().isPosZero()) 12315 return N->getOperand(0); 12316 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 12317 if (C->getValueAPF().isPosZero()) 12318 return N->getOperand(1); 12319 return SDValue(); 12320} 12321 12322static SDValue PerformBTCombine(SDNode *N, 12323 SelectionDAG &DAG, 12324 TargetLowering::DAGCombinerInfo &DCI) { 12325 // BT ignores high bits in the bit index operand. 12326 SDValue Op1 = N->getOperand(1); 12327 if (Op1.hasOneUse()) { 12328 unsigned BitWidth = Op1.getValueSizeInBits(); 12329 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth)); 12330 APInt KnownZero, KnownOne; 12331 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), 12332 !DCI.isBeforeLegalizeOps()); 12333 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 12334 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) || 12335 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO)) 12336 DCI.CommitTargetLoweringOpt(TLO); 12337 } 12338 return SDValue(); 12339} 12340 12341static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) { 12342 SDValue Op = N->getOperand(0); 12343 if (Op.getOpcode() == ISD::BITCAST) 12344 Op = Op.getOperand(0); 12345 EVT VT = N->getValueType(0), OpVT = Op.getValueType(); 12346 if (Op.getOpcode() == X86ISD::VZEXT_LOAD && 12347 VT.getVectorElementType().getSizeInBits() == 12348 OpVT.getVectorElementType().getSizeInBits()) { 12349 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, Op); 12350 } 12351 return SDValue(); 12352} 12353 12354static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG) { 12355 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) -> 12356 // (and (i32 x86isd::setcc_carry), 1) 12357 // This eliminates the zext. This transformation is necessary because 12358 // ISD::SETCC is always legalized to i8. 12359 DebugLoc dl = N->getDebugLoc(); 12360 SDValue N0 = N->getOperand(0); 12361 EVT VT = N->getValueType(0); 12362 if (N0.getOpcode() == ISD::AND && 12363 N0.hasOneUse() && 12364 N0.getOperand(0).hasOneUse()) { 12365 SDValue N00 = N0.getOperand(0); 12366 if (N00.getOpcode() != X86ISD::SETCC_CARRY) 12367 return SDValue(); 12368 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 12369 if (!C || C->getZExtValue() != 1) 12370 return SDValue(); 12371 return DAG.getNode(ISD::AND, dl, VT, 12372 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, 12373 N00.getOperand(0), N00.getOperand(1)), 12374 DAG.getConstant(1, VT)); 12375 } 12376 12377 return SDValue(); 12378} 12379 12380// Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT 12381static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG) { 12382 unsigned X86CC = N->getConstantOperandVal(0); 12383 SDValue EFLAG = N->getOperand(1); 12384 DebugLoc DL = N->getDebugLoc(); 12385 12386 // Materialize "setb reg" as "sbb reg,reg", since it can be extended without 12387 // a zext and produces an all-ones bit which is more useful than 0/1 in some 12388 // cases. 12389 if (X86CC == X86::COND_B) 12390 return DAG.getNode(ISD::AND, DL, MVT::i8, 12391 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, 12392 DAG.getConstant(X86CC, MVT::i8), EFLAG), 12393 DAG.getConstant(1, MVT::i8)); 12394 12395 return SDValue(); 12396} 12397 12398static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, 12399 const X86TargetLowering *XTLI) { 12400 SDValue Op0 = N->getOperand(0); 12401 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have 12402 // a 32-bit target where SSE doesn't support i64->FP operations. 12403 if (Op0.getOpcode() == ISD::LOAD) { 12404 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode()); 12405 EVT VT = Ld->getValueType(0); 12406 if (!Ld->isVolatile() && !N->getValueType(0).isVector() && 12407 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() && 12408 !XTLI->getSubtarget()->is64Bit() && 12409 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) { 12410 SDValue FILDChain = XTLI->BuildFILD(SDValue(N, 0), Ld->getValueType(0), 12411 Ld->getChain(), Op0, DAG); 12412 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1)); 12413 return FILDChain; 12414 } 12415 } 12416 return SDValue(); 12417} 12418 12419// Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS 12420static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG, 12421 X86TargetLowering::DAGCombinerInfo &DCI) { 12422 // If the LHS and RHS of the ADC node are zero, then it can't overflow and 12423 // the result is either zero or one (depending on the input carry bit). 12424 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1. 12425 if (X86::isZeroNode(N->getOperand(0)) && 12426 X86::isZeroNode(N->getOperand(1)) && 12427 // We don't have a good way to replace an EFLAGS use, so only do this when 12428 // dead right now. 12429 SDValue(N, 1).use_empty()) { 12430 DebugLoc DL = N->getDebugLoc(); 12431 EVT VT = N->getValueType(0); 12432 SDValue CarryOut = DAG.getConstant(0, N->getValueType(1)); 12433 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT, 12434 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, 12435 DAG.getConstant(X86::COND_B,MVT::i8), 12436 N->getOperand(2)), 12437 DAG.getConstant(1, VT)); 12438 return DCI.CombineTo(N, Res1, CarryOut); 12439 } 12440 12441 return SDValue(); 12442} 12443 12444// fold (add Y, (sete X, 0)) -> adc 0, Y 12445// (add Y, (setne X, 0)) -> sbb -1, Y 12446// (sub (sete X, 0), Y) -> sbb 0, Y 12447// (sub (setne X, 0), Y) -> adc -1, Y 12448static SDValue OptimizeConditonalInDecrement(SDNode *N, SelectionDAG &DAG) { 12449 DebugLoc DL = N->getDebugLoc(); 12450 12451 // Look through ZExts. 12452 SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0); 12453 if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse()) 12454 return SDValue(); 12455 12456 SDValue SetCC = Ext.getOperand(0); 12457 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse()) 12458 return SDValue(); 12459 12460 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0); 12461 if (CC != X86::COND_E && CC != X86::COND_NE) 12462 return SDValue(); 12463 12464 SDValue Cmp = SetCC.getOperand(1); 12465 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() || 12466 !X86::isZeroNode(Cmp.getOperand(1)) || 12467 !Cmp.getOperand(0).getValueType().isInteger()) 12468 return SDValue(); 12469 12470 SDValue CmpOp0 = Cmp.getOperand(0); 12471 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, 12472 DAG.getConstant(1, CmpOp0.getValueType())); 12473 12474 SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1); 12475 if (CC == X86::COND_NE) 12476 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB, 12477 DL, OtherVal.getValueType(), OtherVal, 12478 DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp); 12479 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC, 12480 DL, OtherVal.getValueType(), OtherVal, 12481 DAG.getConstant(0, OtherVal.getValueType()), NewCmp); 12482} 12483 12484SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 12485 DAGCombinerInfo &DCI) const { 12486 SelectionDAG &DAG = DCI.DAG; 12487 switch (N->getOpcode()) { 12488 default: break; 12489 case ISD::EXTRACT_VECTOR_ELT: 12490 return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, *this); 12491 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 12492 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI); 12493 case ISD::ADD: 12494 case ISD::SUB: return OptimizeConditonalInDecrement(N, DAG); 12495 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI); 12496 case ISD::MUL: return PerformMulCombine(N, DAG, DCI); 12497 case ISD::SHL: 12498 case ISD::SRA: 12499 case ISD::SRL: return PerformShiftCombine(N, DAG, Subtarget); 12500 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget); 12501 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget); 12502 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 12503 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, this); 12504 case X86ISD::FXOR: 12505 case X86ISD::FOR: return PerformFORCombine(N, DAG); 12506 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 12507 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI); 12508 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG); 12509 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG); 12510 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG); 12511 case X86ISD::SHUFPS: // Handle all target specific shuffles 12512 case X86ISD::SHUFPD: 12513 case X86ISD::PALIGN: 12514 case X86ISD::PUNPCKHBW: 12515 case X86ISD::PUNPCKHWD: 12516 case X86ISD::PUNPCKHDQ: 12517 case X86ISD::PUNPCKHQDQ: 12518 case X86ISD::UNPCKHPS: 12519 case X86ISD::UNPCKHPD: 12520 case X86ISD::PUNPCKLBW: 12521 case X86ISD::PUNPCKLWD: 12522 case X86ISD::PUNPCKLDQ: 12523 case X86ISD::PUNPCKLQDQ: 12524 case X86ISD::UNPCKLPS: 12525 case X86ISD::UNPCKLPD: 12526 case X86ISD::VUNPCKLPSY: 12527 case X86ISD::VUNPCKLPDY: 12528 case X86ISD::MOVHLPS: 12529 case X86ISD::MOVLHPS: 12530 case X86ISD::PSHUFD: 12531 case X86ISD::PSHUFHW: 12532 case X86ISD::PSHUFLW: 12533 case X86ISD::MOVSS: 12534 case X86ISD::MOVSD: 12535 case X86ISD::VPERMIL: 12536 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI); 12537 } 12538 12539 return SDValue(); 12540} 12541 12542/// isTypeDesirableForOp - Return true if the target has native support for 12543/// the specified value type and it is 'desirable' to use the type for the 12544/// given node type. e.g. On x86 i16 is legal, but undesirable since i16 12545/// instruction encodings are longer and some i16 instructions are slow. 12546bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const { 12547 if (!isTypeLegal(VT)) 12548 return false; 12549 if (VT != MVT::i16) 12550 return true; 12551 12552 switch (Opc) { 12553 default: 12554 return true; 12555 case ISD::LOAD: 12556 case ISD::SIGN_EXTEND: 12557 case ISD::ZERO_EXTEND: 12558 case ISD::ANY_EXTEND: 12559 case ISD::SHL: 12560 case ISD::SRL: 12561 case ISD::SUB: 12562 case ISD::ADD: 12563 case ISD::MUL: 12564 case ISD::AND: 12565 case ISD::OR: 12566 case ISD::XOR: 12567 return false; 12568 } 12569} 12570 12571/// IsDesirableToPromoteOp - This method query the target whether it is 12572/// beneficial for dag combiner to promote the specified node. If true, it 12573/// should return the desired promotion type by reference. 12574bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const { 12575 EVT VT = Op.getValueType(); 12576 if (VT != MVT::i16) 12577 return false; 12578 12579 bool Promote = false; 12580 bool Commute = false; 12581 switch (Op.getOpcode()) { 12582 default: break; 12583 case ISD::LOAD: { 12584 LoadSDNode *LD = cast<LoadSDNode>(Op); 12585 // If the non-extending load has a single use and it's not live out, then it 12586 // might be folded. 12587 if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&& 12588 Op.hasOneUse()*/) { 12589 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 12590 UE = Op.getNode()->use_end(); UI != UE; ++UI) { 12591 // The only case where we'd want to promote LOAD (rather then it being 12592 // promoted as an operand is when it's only use is liveout. 12593 if (UI->getOpcode() != ISD::CopyToReg) 12594 return false; 12595 } 12596 } 12597 Promote = true; 12598 break; 12599 } 12600 case ISD::SIGN_EXTEND: 12601 case ISD::ZERO_EXTEND: 12602 case ISD::ANY_EXTEND: 12603 Promote = true; 12604 break; 12605 case ISD::SHL: 12606 case ISD::SRL: { 12607 SDValue N0 = Op.getOperand(0); 12608 // Look out for (store (shl (load), x)). 12609 if (MayFoldLoad(N0) && MayFoldIntoStore(Op)) 12610 return false; 12611 Promote = true; 12612 break; 12613 } 12614 case ISD::ADD: 12615 case ISD::MUL: 12616 case ISD::AND: 12617 case ISD::OR: 12618 case ISD::XOR: 12619 Commute = true; 12620 // fallthrough 12621 case ISD::SUB: { 12622 SDValue N0 = Op.getOperand(0); 12623 SDValue N1 = Op.getOperand(1); 12624 if (!Commute && MayFoldLoad(N1)) 12625 return false; 12626 // Avoid disabling potential load folding opportunities. 12627 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op))) 12628 return false; 12629 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op))) 12630 return false; 12631 Promote = true; 12632 } 12633 } 12634 12635 PVT = MVT::i32; 12636 return Promote; 12637} 12638 12639//===----------------------------------------------------------------------===// 12640// X86 Inline Assembly Support 12641//===----------------------------------------------------------------------===// 12642 12643bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const { 12644 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); 12645 12646 std::string AsmStr = IA->getAsmString(); 12647 12648 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a" 12649 SmallVector<StringRef, 4> AsmPieces; 12650 SplitString(AsmStr, AsmPieces, ";\n"); 12651 12652 switch (AsmPieces.size()) { 12653 default: return false; 12654 case 1: 12655 AsmStr = AsmPieces[0]; 12656 AsmPieces.clear(); 12657 SplitString(AsmStr, AsmPieces, " \t"); // Split with whitespace. 12658 12659 // FIXME: this should verify that we are targeting a 486 or better. If not, 12660 // we will turn this bswap into something that will be lowered to logical ops 12661 // instead of emitting the bswap asm. For now, we don't support 486 or lower 12662 // so don't worry about this. 12663 // bswap $0 12664 if (AsmPieces.size() == 2 && 12665 (AsmPieces[0] == "bswap" || 12666 AsmPieces[0] == "bswapq" || 12667 AsmPieces[0] == "bswapl") && 12668 (AsmPieces[1] == "$0" || 12669 AsmPieces[1] == "${0:q}")) { 12670 // No need to check constraints, nothing other than the equivalent of 12671 // "=r,0" would be valid here. 12672 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 12673 if (!Ty || Ty->getBitWidth() % 16 != 0) 12674 return false; 12675 return IntrinsicLowering::LowerToByteSwap(CI); 12676 } 12677 // rorw $$8, ${0:w} --> llvm.bswap.i16 12678 if (CI->getType()->isIntegerTy(16) && 12679 AsmPieces.size() == 3 && 12680 (AsmPieces[0] == "rorw" || AsmPieces[0] == "rolw") && 12681 AsmPieces[1] == "$$8," && 12682 AsmPieces[2] == "${0:w}" && 12683 IA->getConstraintString().compare(0, 5, "=r,0,") == 0) { 12684 AsmPieces.clear(); 12685 const std::string &ConstraintsStr = IA->getConstraintString(); 12686 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ","); 12687 std::sort(AsmPieces.begin(), AsmPieces.end()); 12688 if (AsmPieces.size() == 4 && 12689 AsmPieces[0] == "~{cc}" && 12690 AsmPieces[1] == "~{dirflag}" && 12691 AsmPieces[2] == "~{flags}" && 12692 AsmPieces[3] == "~{fpsr}") { 12693 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 12694 if (!Ty || Ty->getBitWidth() % 16 != 0) 12695 return false; 12696 return IntrinsicLowering::LowerToByteSwap(CI); 12697 } 12698 } 12699 break; 12700 case 3: 12701 if (CI->getType()->isIntegerTy(32) && 12702 IA->getConstraintString().compare(0, 5, "=r,0,") == 0) { 12703 SmallVector<StringRef, 4> Words; 12704 SplitString(AsmPieces[0], Words, " \t,"); 12705 if (Words.size() == 3 && Words[0] == "rorw" && Words[1] == "$$8" && 12706 Words[2] == "${0:w}") { 12707 Words.clear(); 12708 SplitString(AsmPieces[1], Words, " \t,"); 12709 if (Words.size() == 3 && Words[0] == "rorl" && Words[1] == "$$16" && 12710 Words[2] == "$0") { 12711 Words.clear(); 12712 SplitString(AsmPieces[2], Words, " \t,"); 12713 if (Words.size() == 3 && Words[0] == "rorw" && Words[1] == "$$8" && 12714 Words[2] == "${0:w}") { 12715 AsmPieces.clear(); 12716 const std::string &ConstraintsStr = IA->getConstraintString(); 12717 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ","); 12718 std::sort(AsmPieces.begin(), AsmPieces.end()); 12719 if (AsmPieces.size() == 4 && 12720 AsmPieces[0] == "~{cc}" && 12721 AsmPieces[1] == "~{dirflag}" && 12722 AsmPieces[2] == "~{flags}" && 12723 AsmPieces[3] == "~{fpsr}") { 12724 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 12725 if (!Ty || Ty->getBitWidth() % 16 != 0) 12726 return false; 12727 return IntrinsicLowering::LowerToByteSwap(CI); 12728 } 12729 } 12730 } 12731 } 12732 } 12733 12734 if (CI->getType()->isIntegerTy(64)) { 12735 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints(); 12736 if (Constraints.size() >= 2 && 12737 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" && 12738 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") { 12739 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64 12740 SmallVector<StringRef, 4> Words; 12741 SplitString(AsmPieces[0], Words, " \t"); 12742 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%eax") { 12743 Words.clear(); 12744 SplitString(AsmPieces[1], Words, " \t"); 12745 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%edx") { 12746 Words.clear(); 12747 SplitString(AsmPieces[2], Words, " \t,"); 12748 if (Words.size() == 3 && Words[0] == "xchgl" && Words[1] == "%eax" && 12749 Words[2] == "%edx") { 12750 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 12751 if (!Ty || Ty->getBitWidth() % 16 != 0) 12752 return false; 12753 return IntrinsicLowering::LowerToByteSwap(CI); 12754 } 12755 } 12756 } 12757 } 12758 } 12759 break; 12760 } 12761 return false; 12762} 12763 12764 12765 12766/// getConstraintType - Given a constraint letter, return the type of 12767/// constraint it is for this target. 12768X86TargetLowering::ConstraintType 12769X86TargetLowering::getConstraintType(const std::string &Constraint) const { 12770 if (Constraint.size() == 1) { 12771 switch (Constraint[0]) { 12772 case 'R': 12773 case 'q': 12774 case 'Q': 12775 case 'f': 12776 case 't': 12777 case 'u': 12778 case 'y': 12779 case 'x': 12780 case 'Y': 12781 case 'l': 12782 return C_RegisterClass; 12783 case 'a': 12784 case 'b': 12785 case 'c': 12786 case 'd': 12787 case 'S': 12788 case 'D': 12789 case 'A': 12790 return C_Register; 12791 case 'I': 12792 case 'J': 12793 case 'K': 12794 case 'L': 12795 case 'M': 12796 case 'N': 12797 case 'G': 12798 case 'C': 12799 case 'e': 12800 case 'Z': 12801 return C_Other; 12802 default: 12803 break; 12804 } 12805 } 12806 return TargetLowering::getConstraintType(Constraint); 12807} 12808 12809/// Examine constraint type and operand type and determine a weight value. 12810/// This object must already have been set up with the operand type 12811/// and the current alternative constraint selected. 12812TargetLowering::ConstraintWeight 12813 X86TargetLowering::getSingleConstraintMatchWeight( 12814 AsmOperandInfo &info, const char *constraint) const { 12815 ConstraintWeight weight = CW_Invalid; 12816 Value *CallOperandVal = info.CallOperandVal; 12817 // If we don't have a value, we can't do a match, 12818 // but allow it at the lowest weight. 12819 if (CallOperandVal == NULL) 12820 return CW_Default; 12821 Type *type = CallOperandVal->getType(); 12822 // Look at the constraint type. 12823 switch (*constraint) { 12824 default: 12825 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 12826 case 'R': 12827 case 'q': 12828 case 'Q': 12829 case 'a': 12830 case 'b': 12831 case 'c': 12832 case 'd': 12833 case 'S': 12834 case 'D': 12835 case 'A': 12836 if (CallOperandVal->getType()->isIntegerTy()) 12837 weight = CW_SpecificReg; 12838 break; 12839 case 'f': 12840 case 't': 12841 case 'u': 12842 if (type->isFloatingPointTy()) 12843 weight = CW_SpecificReg; 12844 break; 12845 case 'y': 12846 if (type->isX86_MMXTy() && Subtarget->hasMMX()) 12847 weight = CW_SpecificReg; 12848 break; 12849 case 'x': 12850 case 'Y': 12851 if ((type->getPrimitiveSizeInBits() == 128) && Subtarget->hasXMM()) 12852 weight = CW_Register; 12853 break; 12854 case 'I': 12855 if (ConstantInt *C = dyn_cast<ConstantInt>(info.CallOperandVal)) { 12856 if (C->getZExtValue() <= 31) 12857 weight = CW_Constant; 12858 } 12859 break; 12860 case 'J': 12861 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12862 if (C->getZExtValue() <= 63) 12863 weight = CW_Constant; 12864 } 12865 break; 12866 case 'K': 12867 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12868 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f)) 12869 weight = CW_Constant; 12870 } 12871 break; 12872 case 'L': 12873 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12874 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff)) 12875 weight = CW_Constant; 12876 } 12877 break; 12878 case 'M': 12879 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12880 if (C->getZExtValue() <= 3) 12881 weight = CW_Constant; 12882 } 12883 break; 12884 case 'N': 12885 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12886 if (C->getZExtValue() <= 0xff) 12887 weight = CW_Constant; 12888 } 12889 break; 12890 case 'G': 12891 case 'C': 12892 if (dyn_cast<ConstantFP>(CallOperandVal)) { 12893 weight = CW_Constant; 12894 } 12895 break; 12896 case 'e': 12897 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12898 if ((C->getSExtValue() >= -0x80000000LL) && 12899 (C->getSExtValue() <= 0x7fffffffLL)) 12900 weight = CW_Constant; 12901 } 12902 break; 12903 case 'Z': 12904 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 12905 if (C->getZExtValue() <= 0xffffffff) 12906 weight = CW_Constant; 12907 } 12908 break; 12909 } 12910 return weight; 12911} 12912 12913/// LowerXConstraint - try to replace an X constraint, which matches anything, 12914/// with another that has more specific requirements based on the type of the 12915/// corresponding operand. 12916const char *X86TargetLowering:: 12917LowerXConstraint(EVT ConstraintVT) const { 12918 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 12919 // 'f' like normal targets. 12920 if (ConstraintVT.isFloatingPoint()) { 12921 if (Subtarget->hasXMMInt()) 12922 return "Y"; 12923 if (Subtarget->hasXMM()) 12924 return "x"; 12925 } 12926 12927 return TargetLowering::LowerXConstraint(ConstraintVT); 12928} 12929 12930/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 12931/// vector. If it is invalid, don't add anything to Ops. 12932void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 12933 std::string &Constraint, 12934 std::vector<SDValue>&Ops, 12935 SelectionDAG &DAG) const { 12936 SDValue Result(0, 0); 12937 12938 // Only support length 1 constraints for now. 12939 if (Constraint.length() > 1) return; 12940 12941 char ConstraintLetter = Constraint[0]; 12942 switch (ConstraintLetter) { 12943 default: break; 12944 case 'I': 12945 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12946 if (C->getZExtValue() <= 31) { 12947 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 12948 break; 12949 } 12950 } 12951 return; 12952 case 'J': 12953 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12954 if (C->getZExtValue() <= 63) { 12955 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 12956 break; 12957 } 12958 } 12959 return; 12960 case 'K': 12961 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12962 if ((int8_t)C->getSExtValue() == C->getSExtValue()) { 12963 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 12964 break; 12965 } 12966 } 12967 return; 12968 case 'N': 12969 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12970 if (C->getZExtValue() <= 255) { 12971 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 12972 break; 12973 } 12974 } 12975 return; 12976 case 'e': { 12977 // 32-bit signed value 12978 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12979 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 12980 C->getSExtValue())) { 12981 // Widen to 64 bits here to get it sign extended. 12982 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); 12983 break; 12984 } 12985 // FIXME gcc accepts some relocatable values here too, but only in certain 12986 // memory models; it's complicated. 12987 } 12988 return; 12989 } 12990 case 'Z': { 12991 // 32-bit unsigned value 12992 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 12993 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 12994 C->getZExtValue())) { 12995 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 12996 break; 12997 } 12998 } 12999 // FIXME gcc accepts some relocatable values here too, but only in certain 13000 // memory models; it's complicated. 13001 return; 13002 } 13003 case 'i': { 13004 // Literal immediates are always ok. 13005 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 13006 // Widen to 64 bits here to get it sign extended. 13007 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); 13008 break; 13009 } 13010 13011 // In any sort of PIC mode addresses need to be computed at runtime by 13012 // adding in a register or some sort of table lookup. These can't 13013 // be used as immediates. 13014 if (Subtarget->isPICStyleGOT() || Subtarget->isPICStyleStubPIC()) 13015 return; 13016 13017 // If we are in non-pic codegen mode, we allow the address of a global (with 13018 // an optional displacement) to be used with 'i'. 13019 GlobalAddressSDNode *GA = 0; 13020 int64_t Offset = 0; 13021 13022 // Match either (GA), (GA+C), (GA+C1+C2), etc. 13023 while (1) { 13024 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) { 13025 Offset += GA->getOffset(); 13026 break; 13027 } else if (Op.getOpcode() == ISD::ADD) { 13028 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 13029 Offset += C->getZExtValue(); 13030 Op = Op.getOperand(0); 13031 continue; 13032 } 13033 } else if (Op.getOpcode() == ISD::SUB) { 13034 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 13035 Offset += -C->getZExtValue(); 13036 Op = Op.getOperand(0); 13037 continue; 13038 } 13039 } 13040 13041 // Otherwise, this isn't something we can handle, reject it. 13042 return; 13043 } 13044 13045 const GlobalValue *GV = GA->getGlobal(); 13046 // If we require an extra load to get this address, as in PIC mode, we 13047 // can't accept it. 13048 if (isGlobalStubReference(Subtarget->ClassifyGlobalReference(GV, 13049 getTargetMachine()))) 13050 return; 13051 13052 Result = DAG.getTargetGlobalAddress(GV, Op.getDebugLoc(), 13053 GA->getValueType(0), Offset); 13054 break; 13055 } 13056 } 13057 13058 if (Result.getNode()) { 13059 Ops.push_back(Result); 13060 return; 13061 } 13062 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 13063} 13064 13065std::pair<unsigned, const TargetRegisterClass*> 13066X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 13067 EVT VT) const { 13068 // First, see if this is a constraint that directly corresponds to an LLVM 13069 // register class. 13070 if (Constraint.size() == 1) { 13071 // GCC Constraint Letters 13072 switch (Constraint[0]) { 13073 default: break; 13074 // TODO: Slight differences here in allocation order and leaving 13075 // RIP in the class. Do they matter any more here than they do 13076 // in the normal allocation? 13077 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode. 13078 if (Subtarget->is64Bit()) { 13079 if (VT == MVT::i32 || VT == MVT::f32) 13080 return std::make_pair(0U, X86::GR32RegisterClass); 13081 else if (VT == MVT::i16) 13082 return std::make_pair(0U, X86::GR16RegisterClass); 13083 else if (VT == MVT::i8 || VT == MVT::i1) 13084 return std::make_pair(0U, X86::GR8RegisterClass); 13085 else if (VT == MVT::i64 || VT == MVT::f64) 13086 return std::make_pair(0U, X86::GR64RegisterClass); 13087 break; 13088 } 13089 // 32-bit fallthrough 13090 case 'Q': // Q_REGS 13091 if (VT == MVT::i32 || VT == MVT::f32) 13092 return std::make_pair(0U, X86::GR32_ABCDRegisterClass); 13093 else if (VT == MVT::i16) 13094 return std::make_pair(0U, X86::GR16_ABCDRegisterClass); 13095 else if (VT == MVT::i8 || VT == MVT::i1) 13096 return std::make_pair(0U, X86::GR8_ABCD_LRegisterClass); 13097 else if (VT == MVT::i64) 13098 return std::make_pair(0U, X86::GR64_ABCDRegisterClass); 13099 break; 13100 case 'r': // GENERAL_REGS 13101 case 'l': // INDEX_REGS 13102 if (VT == MVT::i8 || VT == MVT::i1) 13103 return std::make_pair(0U, X86::GR8RegisterClass); 13104 if (VT == MVT::i16) 13105 return std::make_pair(0U, X86::GR16RegisterClass); 13106 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit()) 13107 return std::make_pair(0U, X86::GR32RegisterClass); 13108 return std::make_pair(0U, X86::GR64RegisterClass); 13109 case 'R': // LEGACY_REGS 13110 if (VT == MVT::i8 || VT == MVT::i1) 13111 return std::make_pair(0U, X86::GR8_NOREXRegisterClass); 13112 if (VT == MVT::i16) 13113 return std::make_pair(0U, X86::GR16_NOREXRegisterClass); 13114 if (VT == MVT::i32 || !Subtarget->is64Bit()) 13115 return std::make_pair(0U, X86::GR32_NOREXRegisterClass); 13116 return std::make_pair(0U, X86::GR64_NOREXRegisterClass); 13117 case 'f': // FP Stack registers. 13118 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 13119 // value to the correct fpstack register class. 13120 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 13121 return std::make_pair(0U, X86::RFP32RegisterClass); 13122 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 13123 return std::make_pair(0U, X86::RFP64RegisterClass); 13124 return std::make_pair(0U, X86::RFP80RegisterClass); 13125 case 'y': // MMX_REGS if MMX allowed. 13126 if (!Subtarget->hasMMX()) break; 13127 return std::make_pair(0U, X86::VR64RegisterClass); 13128 case 'Y': // SSE_REGS if SSE2 allowed 13129 if (!Subtarget->hasXMMInt()) break; 13130 // FALL THROUGH. 13131 case 'x': // SSE_REGS if SSE1 allowed 13132 if (!Subtarget->hasXMM()) break; 13133 13134 switch (VT.getSimpleVT().SimpleTy) { 13135 default: break; 13136 // Scalar SSE types. 13137 case MVT::f32: 13138 case MVT::i32: 13139 return std::make_pair(0U, X86::FR32RegisterClass); 13140 case MVT::f64: 13141 case MVT::i64: 13142 return std::make_pair(0U, X86::FR64RegisterClass); 13143 // Vector types. 13144 case MVT::v16i8: 13145 case MVT::v8i16: 13146 case MVT::v4i32: 13147 case MVT::v2i64: 13148 case MVT::v4f32: 13149 case MVT::v2f64: 13150 return std::make_pair(0U, X86::VR128RegisterClass); 13151 } 13152 break; 13153 } 13154 } 13155 13156 // Use the default implementation in TargetLowering to convert the register 13157 // constraint into a member of a register class. 13158 std::pair<unsigned, const TargetRegisterClass*> Res; 13159 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 13160 13161 // Not found as a standard register? 13162 if (Res.second == 0) { 13163 // Map st(0) -> st(7) -> ST0 13164 if (Constraint.size() == 7 && Constraint[0] == '{' && 13165 tolower(Constraint[1]) == 's' && 13166 tolower(Constraint[2]) == 't' && 13167 Constraint[3] == '(' && 13168 (Constraint[4] >= '0' && Constraint[4] <= '7') && 13169 Constraint[5] == ')' && 13170 Constraint[6] == '}') { 13171 13172 Res.first = X86::ST0+Constraint[4]-'0'; 13173 Res.second = X86::RFP80RegisterClass; 13174 return Res; 13175 } 13176 13177 // GCC allows "st(0)" to be called just plain "st". 13178 if (StringRef("{st}").equals_lower(Constraint)) { 13179 Res.first = X86::ST0; 13180 Res.second = X86::RFP80RegisterClass; 13181 return Res; 13182 } 13183 13184 // flags -> EFLAGS 13185 if (StringRef("{flags}").equals_lower(Constraint)) { 13186 Res.first = X86::EFLAGS; 13187 Res.second = X86::CCRRegisterClass; 13188 return Res; 13189 } 13190 13191 // 'A' means EAX + EDX. 13192 if (Constraint == "A") { 13193 Res.first = X86::EAX; 13194 Res.second = X86::GR32_ADRegisterClass; 13195 return Res; 13196 } 13197 return Res; 13198 } 13199 13200 // Otherwise, check to see if this is a register class of the wrong value 13201 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 13202 // turn into {ax},{dx}. 13203 if (Res.second->hasType(VT)) 13204 return Res; // Correct type already, nothing to do. 13205 13206 // All of the single-register GCC register classes map their values onto 13207 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 13208 // really want an 8-bit or 32-bit register, map to the appropriate register 13209 // class and return the appropriate register. 13210 if (Res.second == X86::GR16RegisterClass) { 13211 if (VT == MVT::i8) { 13212 unsigned DestReg = 0; 13213 switch (Res.first) { 13214 default: break; 13215 case X86::AX: DestReg = X86::AL; break; 13216 case X86::DX: DestReg = X86::DL; break; 13217 case X86::CX: DestReg = X86::CL; break; 13218 case X86::BX: DestReg = X86::BL; break; 13219 } 13220 if (DestReg) { 13221 Res.first = DestReg; 13222 Res.second = X86::GR8RegisterClass; 13223 } 13224 } else if (VT == MVT::i32) { 13225 unsigned DestReg = 0; 13226 switch (Res.first) { 13227 default: break; 13228 case X86::AX: DestReg = X86::EAX; break; 13229 case X86::DX: DestReg = X86::EDX; break; 13230 case X86::CX: DestReg = X86::ECX; break; 13231 case X86::BX: DestReg = X86::EBX; break; 13232 case X86::SI: DestReg = X86::ESI; break; 13233 case X86::DI: DestReg = X86::EDI; break; 13234 case X86::BP: DestReg = X86::EBP; break; 13235 case X86::SP: DestReg = X86::ESP; break; 13236 } 13237 if (DestReg) { 13238 Res.first = DestReg; 13239 Res.second = X86::GR32RegisterClass; 13240 } 13241 } else if (VT == MVT::i64) { 13242 unsigned DestReg = 0; 13243 switch (Res.first) { 13244 default: break; 13245 case X86::AX: DestReg = X86::RAX; break; 13246 case X86::DX: DestReg = X86::RDX; break; 13247 case X86::CX: DestReg = X86::RCX; break; 13248 case X86::BX: DestReg = X86::RBX; break; 13249 case X86::SI: DestReg = X86::RSI; break; 13250 case X86::DI: DestReg = X86::RDI; break; 13251 case X86::BP: DestReg = X86::RBP; break; 13252 case X86::SP: DestReg = X86::RSP; break; 13253 } 13254 if (DestReg) { 13255 Res.first = DestReg; 13256 Res.second = X86::GR64RegisterClass; 13257 } 13258 } 13259 } else if (Res.second == X86::FR32RegisterClass || 13260 Res.second == X86::FR64RegisterClass || 13261 Res.second == X86::VR128RegisterClass) { 13262 // Handle references to XMM physical registers that got mapped into the 13263 // wrong class. This can happen with constraints like {xmm0} where the 13264 // target independent register mapper will just pick the first match it can 13265 // find, ignoring the required type. 13266 if (VT == MVT::f32) 13267 Res.second = X86::FR32RegisterClass; 13268 else if (VT == MVT::f64) 13269 Res.second = X86::FR64RegisterClass; 13270 else if (X86::VR128RegisterClass->hasType(VT)) 13271 Res.second = X86::VR128RegisterClass; 13272 } 13273 13274 return Res; 13275} 13276