X86ISelLowering.cpp revision d978c54e607fbcf426db20727d5fed71e1def2f6
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "x86-isel" 16#include "X86ISelLowering.h" 17#include "X86.h" 18#include "X86InstrBuilder.h" 19#include "X86TargetMachine.h" 20#include "X86TargetObjectFile.h" 21#include "Utils/X86ShuffleDecode.h" 22#include "llvm/CallingConv.h" 23#include "llvm/Constants.h" 24#include "llvm/DerivedTypes.h" 25#include "llvm/GlobalAlias.h" 26#include "llvm/GlobalVariable.h" 27#include "llvm/Function.h" 28#include "llvm/Instructions.h" 29#include "llvm/Intrinsics.h" 30#include "llvm/LLVMContext.h" 31#include "llvm/CodeGen/IntrinsicLowering.h" 32#include "llvm/CodeGen/MachineFrameInfo.h" 33#include "llvm/CodeGen/MachineFunction.h" 34#include "llvm/CodeGen/MachineInstrBuilder.h" 35#include "llvm/CodeGen/MachineJumpTableInfo.h" 36#include "llvm/CodeGen/MachineModuleInfo.h" 37#include "llvm/CodeGen/MachineRegisterInfo.h" 38#include "llvm/MC/MCAsmInfo.h" 39#include "llvm/MC/MCContext.h" 40#include "llvm/MC/MCExpr.h" 41#include "llvm/MC/MCSymbol.h" 42#include "llvm/ADT/SmallSet.h" 43#include "llvm/ADT/Statistic.h" 44#include "llvm/ADT/StringExtras.h" 45#include "llvm/ADT/VariadicFunction.h" 46#include "llvm/Support/CallSite.h" 47#include "llvm/Support/Debug.h" 48#include "llvm/Support/ErrorHandling.h" 49#include "llvm/Support/MathExtras.h" 50#include "llvm/Target/TargetOptions.h" 51#include <bitset> 52using namespace llvm; 53 54STATISTIC(NumTailCalls, "Number of tail calls"); 55 56// Forward declarations. 57static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 58 SDValue V2); 59 60/// Generate a DAG to grab 128-bits from a vector > 128 bits. This 61/// sets things up to match to an AVX VEXTRACTF128 instruction or a 62/// simple subregister reference. Idx is an index in the 128 bits we 63/// want. It need not be aligned to a 128-bit bounday. That makes 64/// lowering EXTRACT_VECTOR_ELT operations easier. 65static SDValue Extract128BitVector(SDValue Vec, unsigned IdxVal, 66 SelectionDAG &DAG, DebugLoc dl) { 67 EVT VT = Vec.getValueType(); 68 assert(VT.getSizeInBits() == 256 && "Unexpected vector size!"); 69 EVT ElVT = VT.getVectorElementType(); 70 unsigned Factor = VT.getSizeInBits()/128; 71 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, 72 VT.getVectorNumElements()/Factor); 73 74 // Extract from UNDEF is UNDEF. 75 if (Vec.getOpcode() == ISD::UNDEF) 76 return DAG.getUNDEF(ResultVT); 77 78 // Extract the relevant 128 bits. Generate an EXTRACT_SUBVECTOR 79 // we can match to VEXTRACTF128. 80 unsigned ElemsPerChunk = 128 / ElVT.getSizeInBits(); 81 82 // This is the index of the first element of the 128-bit chunk 83 // we want. 84 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / 128) 85 * ElemsPerChunk); 86 87 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); 88 SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, 89 VecIdx); 90 91 return Result; 92} 93 94/// Generate a DAG to put 128-bits into a vector > 128 bits. This 95/// sets things up to match to an AVX VINSERTF128 instruction or a 96/// simple superregister reference. Idx is an index in the 128 bits 97/// we want. It need not be aligned to a 128-bit bounday. That makes 98/// lowering INSERT_VECTOR_ELT operations easier. 99static SDValue Insert128BitVector(SDValue Result, SDValue Vec, 100 unsigned IdxVal, SelectionDAG &DAG, 101 DebugLoc dl) { 102 EVT VT = Vec.getValueType(); 103 assert(VT.getSizeInBits() == 128 && "Unexpected vector size!"); 104 105 EVT ElVT = VT.getVectorElementType(); 106 EVT ResultVT = Result.getValueType(); 107 108 // Insert the relevant 128 bits. 109 unsigned ElemsPerChunk = 128/ElVT.getSizeInBits(); 110 111 // This is the index of the first element of the 128-bit chunk 112 // we want. 113 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/128) 114 * ElemsPerChunk); 115 116 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); 117 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, 118 VecIdx); 119 return Result; 120} 121 122/// Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 123/// instructions. This is used because creating CONCAT_VECTOR nodes of 124/// BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower 125/// large BUILD_VECTORS. 126static SDValue Concat128BitVectors(SDValue V1, SDValue V2, EVT VT, 127 unsigned NumElems, SelectionDAG &DAG, 128 DebugLoc dl) { 129 SDValue V = Insert128BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl); 130 return Insert128BitVector(V, V2, NumElems/2, DAG, dl); 131} 132 133static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) { 134 const X86Subtarget *Subtarget = &TM.getSubtarget<X86Subtarget>(); 135 bool is64Bit = Subtarget->is64Bit(); 136 137 if (Subtarget->isTargetEnvMacho()) { 138 if (is64Bit) 139 return new X8664_MachoTargetObjectFile(); 140 return new TargetLoweringObjectFileMachO(); 141 } 142 143 if (Subtarget->isTargetELF()) 144 return new TargetLoweringObjectFileELF(); 145 if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) 146 return new TargetLoweringObjectFileCOFF(); 147 llvm_unreachable("unknown subtarget type"); 148} 149 150X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 151 : TargetLowering(TM, createTLOF(TM)) { 152 Subtarget = &TM.getSubtarget<X86Subtarget>(); 153 X86ScalarSSEf64 = Subtarget->hasSSE2(); 154 X86ScalarSSEf32 = Subtarget->hasSSE1(); 155 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 156 157 RegInfo = TM.getRegisterInfo(); 158 TD = getTargetData(); 159 160 // Set up the TargetLowering object. 161 static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }; 162 163 // X86 is weird, it always uses i8 for shift amounts and setcc results. 164 setBooleanContents(ZeroOrOneBooleanContent); 165 // X86-SSE is even stranger. It uses -1 or 0 for vector masks. 166 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 167 168 // For 64-bit since we have so many registers use the ILP scheduler, for 169 // 32-bit code use the register pressure specific scheduling. 170 // For Atom, always use ILP scheduling. 171 if (Subtarget->isAtom()) 172 setSchedulingPreference(Sched::ILP); 173 else if (Subtarget->is64Bit()) 174 setSchedulingPreference(Sched::ILP); 175 else 176 setSchedulingPreference(Sched::RegPressure); 177 setStackPointerRegisterToSaveRestore(X86StackPtr); 178 179 if (Subtarget->isTargetWindows() && !Subtarget->isTargetCygMing()) { 180 // Setup Windows compiler runtime calls. 181 setLibcallName(RTLIB::SDIV_I64, "_alldiv"); 182 setLibcallName(RTLIB::UDIV_I64, "_aulldiv"); 183 setLibcallName(RTLIB::SREM_I64, "_allrem"); 184 setLibcallName(RTLIB::UREM_I64, "_aullrem"); 185 setLibcallName(RTLIB::MUL_I64, "_allmul"); 186 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall); 187 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall); 188 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall); 189 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall); 190 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall); 191 192 // The _ftol2 runtime function has an unusual calling conv, which 193 // is modeled by a special pseudo-instruction. 194 setLibcallName(RTLIB::FPTOUINT_F64_I64, 0); 195 setLibcallName(RTLIB::FPTOUINT_F32_I64, 0); 196 setLibcallName(RTLIB::FPTOUINT_F64_I32, 0); 197 setLibcallName(RTLIB::FPTOUINT_F32_I32, 0); 198 } 199 200 if (Subtarget->isTargetDarwin()) { 201 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 202 setUseUnderscoreSetJmp(false); 203 setUseUnderscoreLongJmp(false); 204 } else if (Subtarget->isTargetMingw()) { 205 // MS runtime is weird: it exports _setjmp, but longjmp! 206 setUseUnderscoreSetJmp(true); 207 setUseUnderscoreLongJmp(false); 208 } else { 209 setUseUnderscoreSetJmp(true); 210 setUseUnderscoreLongJmp(true); 211 } 212 213 // Set up the register classes. 214 addRegisterClass(MVT::i8, &X86::GR8RegClass); 215 addRegisterClass(MVT::i16, &X86::GR16RegClass); 216 addRegisterClass(MVT::i32, &X86::GR32RegClass); 217 if (Subtarget->is64Bit()) 218 addRegisterClass(MVT::i64, &X86::GR64RegClass); 219 220 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 221 222 // We don't accept any truncstore of integer registers. 223 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 224 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 225 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 226 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 227 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 228 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 229 230 // SETOEQ and SETUNE require checking two conditions. 231 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); 232 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); 233 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); 234 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); 235 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); 236 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); 237 238 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 239 // operation. 240 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 241 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 242 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 243 244 if (Subtarget->is64Bit()) { 245 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 246 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 247 } else if (!TM.Options.UseSoftFloat) { 248 // We have an algorithm for SSE2->double, and we turn this into a 249 // 64-bit FILD followed by conditional FADD for other targets. 250 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 251 // We have an algorithm for SSE2, and we turn this into a 64-bit 252 // FILD for other targets. 253 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); 254 } 255 256 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 257 // this operation. 258 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 259 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 260 261 if (!TM.Options.UseSoftFloat) { 262 // SSE has no i16 to fp conversion, only i32 263 if (X86ScalarSSEf32) { 264 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 265 // f32 and f64 cases are Legal, f80 case is not 266 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 267 } else { 268 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 269 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 270 } 271 } else { 272 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 273 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); 274 } 275 276 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 277 // are Legal, f80 is custom lowered. 278 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 279 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 280 281 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 282 // this operation. 283 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 284 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 285 286 if (X86ScalarSSEf32) { 287 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 288 // f32 and f64 cases are Legal, f80 case is not 289 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 290 } else { 291 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 292 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 293 } 294 295 // Handle FP_TO_UINT by promoting the destination to a larger signed 296 // conversion. 297 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 298 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 299 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 300 301 if (Subtarget->is64Bit()) { 302 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 303 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 304 } else if (!TM.Options.UseSoftFloat) { 305 // Since AVX is a superset of SSE3, only check for SSE here. 306 if (Subtarget->hasSSE1() && !Subtarget->hasSSE3()) 307 // Expand FP_TO_UINT into a select. 308 // FIXME: We would like to use a Custom expander here eventually to do 309 // the optimal thing for SSE vs. the default expansion in the legalizer. 310 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 311 else 312 // With SSE3 we can use fisttpll to convert to a signed i64; without 313 // SSE, we're stuck with a fistpll. 314 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); 315 } 316 317 if (isTargetFTOL()) { 318 // Use the _ftol2 runtime function, which has a pseudo-instruction 319 // to handle its weird calling convention. 320 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); 321 } 322 323 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 324 if (!X86ScalarSSEf64) { 325 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); 326 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); 327 if (Subtarget->is64Bit()) { 328 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); 329 // Without SSE, i64->f64 goes through memory. 330 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); 331 } 332 } 333 334 // Scalar integer divide and remainder are lowered to use operations that 335 // produce two results, to match the available instructions. This exposes 336 // the two-result form to trivial CSE, which is able to combine x/y and x%y 337 // into a single instruction. 338 // 339 // Scalar integer multiply-high is also lowered to use two-result 340 // operations, to match the available instructions. However, plain multiply 341 // (low) operations are left as Legal, as there are single-result 342 // instructions for this in x86. Using the two-result multiply instructions 343 // when both high and low results are needed must be arranged by dagcombine. 344 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) { 345 MVT VT = IntVTs[i]; 346 setOperationAction(ISD::MULHS, VT, Expand); 347 setOperationAction(ISD::MULHU, VT, Expand); 348 setOperationAction(ISD::SDIV, VT, Expand); 349 setOperationAction(ISD::UDIV, VT, Expand); 350 setOperationAction(ISD::SREM, VT, Expand); 351 setOperationAction(ISD::UREM, VT, Expand); 352 353 // Add/Sub overflow ops with MVT::Glues are lowered to EFLAGS dependences. 354 setOperationAction(ISD::ADDC, VT, Custom); 355 setOperationAction(ISD::ADDE, VT, Custom); 356 setOperationAction(ISD::SUBC, VT, Custom); 357 setOperationAction(ISD::SUBE, VT, Custom); 358 } 359 360 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 361 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 362 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 363 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 364 if (Subtarget->is64Bit()) 365 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 366 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 367 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 368 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 369 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 370 setOperationAction(ISD::FREM , MVT::f32 , Expand); 371 setOperationAction(ISD::FREM , MVT::f64 , Expand); 372 setOperationAction(ISD::FREM , MVT::f80 , Expand); 373 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 374 375 // Promote the i8 variants and force them on up to i32 which has a shorter 376 // encoding. 377 setOperationAction(ISD::CTTZ , MVT::i8 , Promote); 378 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32); 379 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote); 380 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32); 381 if (Subtarget->hasBMI()) { 382 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand); 383 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand); 384 if (Subtarget->is64Bit()) 385 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); 386 } else { 387 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 388 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 389 if (Subtarget->is64Bit()) 390 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 391 } 392 393 if (Subtarget->hasLZCNT()) { 394 // When promoting the i8 variants, force them to i32 for a shorter 395 // encoding. 396 setOperationAction(ISD::CTLZ , MVT::i8 , Promote); 397 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32); 398 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote); 399 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); 400 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand); 401 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand); 402 if (Subtarget->is64Bit()) 403 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); 404 } else { 405 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 406 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 407 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 408 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); 409 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); 410 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); 411 if (Subtarget->is64Bit()) { 412 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 413 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); 414 } 415 } 416 417 if (Subtarget->hasPOPCNT()) { 418 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); 419 } else { 420 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 421 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 422 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 423 if (Subtarget->is64Bit()) 424 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 425 } 426 427 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 428 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 429 430 // These should be promoted to a larger select which is supported. 431 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 432 // X86 wants to expand cmov itself. 433 setOperationAction(ISD::SELECT , MVT::i8 , Custom); 434 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 435 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 436 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 437 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 438 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 439 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 440 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 441 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 442 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 443 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 444 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 445 if (Subtarget->is64Bit()) { 446 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 447 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 448 } 449 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 450 451 // Darwin ABI issue. 452 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 453 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 454 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 455 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 456 if (Subtarget->is64Bit()) 457 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 458 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 459 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); 460 if (Subtarget->is64Bit()) { 461 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 462 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 463 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 464 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 465 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); 466 } 467 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 468 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 469 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 470 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 471 if (Subtarget->is64Bit()) { 472 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 473 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 474 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 475 } 476 477 if (Subtarget->hasSSE1()) 478 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 479 480 setOperationAction(ISD::MEMBARRIER , MVT::Other, Custom); 481 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); 482 483 // On X86 and X86-64, atomic operations are lowered to locked instructions. 484 // Locked instructions, in turn, have implicit fence semantics (all memory 485 // operations are flushed before issuing the locked instruction, and they 486 // are not buffered), so we can fold away the common pattern of 487 // fence-atomic-fence. 488 setShouldFoldAtomicFences(true); 489 490 // Expand certain atomics 491 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) { 492 MVT VT = IntVTs[i]; 493 setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom); 494 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 495 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 496 } 497 498 if (!Subtarget->is64Bit()) { 499 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom); 500 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom); 501 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); 502 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); 503 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom); 504 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom); 505 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom); 506 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom); 507 } 508 509 if (Subtarget->hasCmpxchg16b()) { 510 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); 511 } 512 513 // FIXME - use subtarget debug flags 514 if (!Subtarget->isTargetDarwin() && 515 !Subtarget->isTargetELF() && 516 !Subtarget->isTargetCygMing()) { 517 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 518 } 519 520 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 521 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 522 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 523 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 524 if (Subtarget->is64Bit()) { 525 setExceptionPointerRegister(X86::RAX); 526 setExceptionSelectorRegister(X86::RDX); 527 } else { 528 setExceptionPointerRegister(X86::EAX); 529 setExceptionSelectorRegister(X86::EDX); 530 } 531 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 532 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 533 534 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 535 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 536 537 setOperationAction(ISD::TRAP, MVT::Other, Legal); 538 539 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 540 setOperationAction(ISD::VASTART , MVT::Other, Custom); 541 setOperationAction(ISD::VAEND , MVT::Other, Expand); 542 if (Subtarget->is64Bit()) { 543 setOperationAction(ISD::VAARG , MVT::Other, Custom); 544 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 545 } else { 546 setOperationAction(ISD::VAARG , MVT::Other, Expand); 547 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 548 } 549 550 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 551 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 552 553 if (Subtarget->isTargetCOFF() && !Subtarget->isTargetEnvMacho()) 554 setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ? 555 MVT::i64 : MVT::i32, Custom); 556 else if (TM.Options.EnableSegmentedStacks) 557 setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ? 558 MVT::i64 : MVT::i32, Custom); 559 else 560 setOperationAction(ISD::DYNAMIC_STACKALLOC, Subtarget->is64Bit() ? 561 MVT::i64 : MVT::i32, Expand); 562 563 if (!TM.Options.UseSoftFloat && X86ScalarSSEf64) { 564 // f32 and f64 use SSE. 565 // Set up the FP register classes. 566 addRegisterClass(MVT::f32, &X86::FR32RegClass); 567 addRegisterClass(MVT::f64, &X86::FR64RegClass); 568 569 // Use ANDPD to simulate FABS. 570 setOperationAction(ISD::FABS , MVT::f64, Custom); 571 setOperationAction(ISD::FABS , MVT::f32, Custom); 572 573 // Use XORP to simulate FNEG. 574 setOperationAction(ISD::FNEG , MVT::f64, Custom); 575 setOperationAction(ISD::FNEG , MVT::f32, Custom); 576 577 // Use ANDPD and ORPD to simulate FCOPYSIGN. 578 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 579 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 580 581 // Lower this to FGETSIGNx86 plus an AND. 582 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); 583 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); 584 585 // We don't support sin/cos/fmod 586 setOperationAction(ISD::FSIN , MVT::f64, Expand); 587 setOperationAction(ISD::FCOS , MVT::f64, Expand); 588 setOperationAction(ISD::FSIN , MVT::f32, Expand); 589 setOperationAction(ISD::FCOS , MVT::f32, Expand); 590 591 // Expand FP immediates into loads from the stack, except for the special 592 // cases we handle. 593 addLegalFPImmediate(APFloat(+0.0)); // xorpd 594 addLegalFPImmediate(APFloat(+0.0f)); // xorps 595 } else if (!TM.Options.UseSoftFloat && X86ScalarSSEf32) { 596 // Use SSE for f32, x87 for f64. 597 // Set up the FP register classes. 598 addRegisterClass(MVT::f32, &X86::FR32RegClass); 599 addRegisterClass(MVT::f64, &X86::RFP64RegClass); 600 601 // Use ANDPS to simulate FABS. 602 setOperationAction(ISD::FABS , MVT::f32, Custom); 603 604 // Use XORP to simulate FNEG. 605 setOperationAction(ISD::FNEG , MVT::f32, Custom); 606 607 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 608 609 // Use ANDPS and ORPS to simulate FCOPYSIGN. 610 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 611 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 612 613 // We don't support sin/cos/fmod 614 setOperationAction(ISD::FSIN , MVT::f32, Expand); 615 setOperationAction(ISD::FCOS , MVT::f32, Expand); 616 617 // Special cases we handle for FP constants. 618 addLegalFPImmediate(APFloat(+0.0f)); // xorps 619 addLegalFPImmediate(APFloat(+0.0)); // FLD0 620 addLegalFPImmediate(APFloat(+1.0)); // FLD1 621 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 622 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 623 624 if (!TM.Options.UnsafeFPMath) { 625 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 626 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 627 } 628 } else if (!TM.Options.UseSoftFloat) { 629 // f32 and f64 in x87. 630 // Set up the FP register classes. 631 addRegisterClass(MVT::f64, &X86::RFP64RegClass); 632 addRegisterClass(MVT::f32, &X86::RFP32RegClass); 633 634 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 635 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 636 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 637 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 638 639 if (!TM.Options.UnsafeFPMath) { 640 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 641 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 642 } 643 addLegalFPImmediate(APFloat(+0.0)); // FLD0 644 addLegalFPImmediate(APFloat(+1.0)); // FLD1 645 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 646 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 647 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 648 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 649 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 650 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 651 } 652 653 // We don't support FMA. 654 setOperationAction(ISD::FMA, MVT::f64, Expand); 655 setOperationAction(ISD::FMA, MVT::f32, Expand); 656 657 // Long double always uses X87. 658 if (!TM.Options.UseSoftFloat) { 659 addRegisterClass(MVT::f80, &X86::RFP80RegClass); 660 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 661 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 662 { 663 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended); 664 addLegalFPImmediate(TmpFlt); // FLD0 665 TmpFlt.changeSign(); 666 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 667 668 bool ignored; 669 APFloat TmpFlt2(+1.0); 670 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, 671 &ignored); 672 addLegalFPImmediate(TmpFlt2); // FLD1 673 TmpFlt2.changeSign(); 674 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 675 } 676 677 if (!TM.Options.UnsafeFPMath) { 678 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 679 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 680 } 681 682 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); 683 setOperationAction(ISD::FCEIL, MVT::f80, Expand); 684 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); 685 setOperationAction(ISD::FRINT, MVT::f80, Expand); 686 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); 687 setOperationAction(ISD::FMA, MVT::f80, Expand); 688 } 689 690 // Always use a library call for pow. 691 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 692 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 693 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 694 695 setOperationAction(ISD::FLOG, MVT::f80, Expand); 696 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 697 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 698 setOperationAction(ISD::FEXP, MVT::f80, Expand); 699 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 700 701 // First set operation action for all vector types to either promote 702 // (for widening) or expand (for scalarization). Then we will selectively 703 // turn on ones that can be effectively codegen'd. 704 for (int VT = MVT::FIRST_VECTOR_VALUETYPE; 705 VT <= MVT::LAST_VECTOR_VALUETYPE; ++VT) { 706 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 707 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 708 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 709 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 710 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 711 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 712 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 713 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 714 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 715 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 716 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 717 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 718 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 719 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 720 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); 721 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); 722 setOperationAction(ISD::EXTRACT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand); 723 setOperationAction(ISD::INSERT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand); 724 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 725 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 726 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 727 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 728 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 729 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 730 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 731 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 732 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 733 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 734 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 735 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 736 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 737 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 738 setOperationAction(ISD::CTTZ_ZERO_UNDEF, (MVT::SimpleValueType)VT, Expand); 739 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 740 setOperationAction(ISD::CTLZ_ZERO_UNDEF, (MVT::SimpleValueType)VT, Expand); 741 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 742 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 743 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 744 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 745 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 746 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 747 setOperationAction(ISD::SETCC, (MVT::SimpleValueType)VT, Expand); 748 setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand); 749 setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand); 750 setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand); 751 setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand); 752 setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand); 753 setOperationAction(ISD::FP_TO_UINT, (MVT::SimpleValueType)VT, Expand); 754 setOperationAction(ISD::FP_TO_SINT, (MVT::SimpleValueType)VT, Expand); 755 setOperationAction(ISD::UINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 756 setOperationAction(ISD::SINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 757 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,Expand); 758 setOperationAction(ISD::TRUNCATE, (MVT::SimpleValueType)VT, Expand); 759 setOperationAction(ISD::SIGN_EXTEND, (MVT::SimpleValueType)VT, Expand); 760 setOperationAction(ISD::ZERO_EXTEND, (MVT::SimpleValueType)VT, Expand); 761 setOperationAction(ISD::ANY_EXTEND, (MVT::SimpleValueType)VT, Expand); 762 setOperationAction(ISD::VSELECT, (MVT::SimpleValueType)VT, Expand); 763 for (int InnerVT = MVT::FIRST_VECTOR_VALUETYPE; 764 InnerVT <= MVT::LAST_VECTOR_VALUETYPE; ++InnerVT) 765 setTruncStoreAction((MVT::SimpleValueType)VT, 766 (MVT::SimpleValueType)InnerVT, Expand); 767 setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT, Expand); 768 setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT, Expand); 769 setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT, Expand); 770 } 771 772 // FIXME: In order to prevent SSE instructions being expanded to MMX ones 773 // with -msoft-float, disable use of MMX as well. 774 if (!TM.Options.UseSoftFloat && Subtarget->hasMMX()) { 775 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); 776 // No operations on x86mmx supported, everything uses intrinsics. 777 } 778 779 // MMX-sized vectors (other than x86mmx) are expected to be expanded 780 // into smaller operations. 781 setOperationAction(ISD::MULHS, MVT::v8i8, Expand); 782 setOperationAction(ISD::MULHS, MVT::v4i16, Expand); 783 setOperationAction(ISD::MULHS, MVT::v2i32, Expand); 784 setOperationAction(ISD::MULHS, MVT::v1i64, Expand); 785 setOperationAction(ISD::AND, MVT::v8i8, Expand); 786 setOperationAction(ISD::AND, MVT::v4i16, Expand); 787 setOperationAction(ISD::AND, MVT::v2i32, Expand); 788 setOperationAction(ISD::AND, MVT::v1i64, Expand); 789 setOperationAction(ISD::OR, MVT::v8i8, Expand); 790 setOperationAction(ISD::OR, MVT::v4i16, Expand); 791 setOperationAction(ISD::OR, MVT::v2i32, Expand); 792 setOperationAction(ISD::OR, MVT::v1i64, Expand); 793 setOperationAction(ISD::XOR, MVT::v8i8, Expand); 794 setOperationAction(ISD::XOR, MVT::v4i16, Expand); 795 setOperationAction(ISD::XOR, MVT::v2i32, Expand); 796 setOperationAction(ISD::XOR, MVT::v1i64, Expand); 797 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Expand); 798 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Expand); 799 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i32, Expand); 800 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Expand); 801 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); 802 setOperationAction(ISD::SELECT, MVT::v8i8, Expand); 803 setOperationAction(ISD::SELECT, MVT::v4i16, Expand); 804 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 805 setOperationAction(ISD::SELECT, MVT::v1i64, Expand); 806 setOperationAction(ISD::BITCAST, MVT::v8i8, Expand); 807 setOperationAction(ISD::BITCAST, MVT::v4i16, Expand); 808 setOperationAction(ISD::BITCAST, MVT::v2i32, Expand); 809 setOperationAction(ISD::BITCAST, MVT::v1i64, Expand); 810 811 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE1()) { 812 addRegisterClass(MVT::v4f32, &X86::VR128RegClass); 813 814 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 815 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 816 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 817 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 818 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 819 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 820 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 821 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 822 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 823 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 824 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 825 setOperationAction(ISD::SETCC, MVT::v4f32, Custom); 826 } 827 828 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE2()) { 829 addRegisterClass(MVT::v2f64, &X86::VR128RegClass); 830 831 // FIXME: Unfortunately -soft-float and -no-implicit-float means XMM 832 // registers cannot be used even for integer operations. 833 addRegisterClass(MVT::v16i8, &X86::VR128RegClass); 834 addRegisterClass(MVT::v8i16, &X86::VR128RegClass); 835 addRegisterClass(MVT::v4i32, &X86::VR128RegClass); 836 addRegisterClass(MVT::v2i64, &X86::VR128RegClass); 837 838 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 839 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 840 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 841 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 842 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 843 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 844 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 845 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 846 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 847 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 848 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 849 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 850 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 851 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 852 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 853 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 854 855 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 856 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); 857 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); 858 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); 859 860 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 861 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 862 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 863 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 864 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 865 866 setOperationAction(ISD::CONCAT_VECTORS, MVT::v2f64, Custom); 867 setOperationAction(ISD::CONCAT_VECTORS, MVT::v2i64, Custom); 868 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i8, Custom); 869 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i16, Custom); 870 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 871 872 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 873 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { 874 EVT VT = (MVT::SimpleValueType)i; 875 // Do not attempt to custom lower non-power-of-2 vectors 876 if (!isPowerOf2_32(VT.getVectorNumElements())) 877 continue; 878 // Do not attempt to custom lower non-128-bit vectors 879 if (!VT.is128BitVector()) 880 continue; 881 setOperationAction(ISD::BUILD_VECTOR, 882 VT.getSimpleVT().SimpleTy, Custom); 883 setOperationAction(ISD::VECTOR_SHUFFLE, 884 VT.getSimpleVT().SimpleTy, Custom); 885 setOperationAction(ISD::EXTRACT_VECTOR_ELT, 886 VT.getSimpleVT().SimpleTy, Custom); 887 } 888 889 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 890 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 891 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 892 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 893 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 894 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 895 896 if (Subtarget->is64Bit()) { 897 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 898 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 899 } 900 901 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 902 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { 903 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 904 EVT VT = SVT; 905 906 // Do not attempt to promote non-128-bit vectors 907 if (!VT.is128BitVector()) 908 continue; 909 910 setOperationAction(ISD::AND, SVT, Promote); 911 AddPromotedToType (ISD::AND, SVT, MVT::v2i64); 912 setOperationAction(ISD::OR, SVT, Promote); 913 AddPromotedToType (ISD::OR, SVT, MVT::v2i64); 914 setOperationAction(ISD::XOR, SVT, Promote); 915 AddPromotedToType (ISD::XOR, SVT, MVT::v2i64); 916 setOperationAction(ISD::LOAD, SVT, Promote); 917 AddPromotedToType (ISD::LOAD, SVT, MVT::v2i64); 918 setOperationAction(ISD::SELECT, SVT, Promote); 919 AddPromotedToType (ISD::SELECT, SVT, MVT::v2i64); 920 } 921 922 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 923 924 // Custom lower v2i64 and v2f64 selects. 925 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 926 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 927 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 928 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 929 930 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 931 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 932 } 933 934 if (Subtarget->hasSSE41()) { 935 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 936 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 937 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 938 setOperationAction(ISD::FRINT, MVT::f32, Legal); 939 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 940 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 941 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 942 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 943 setOperationAction(ISD::FRINT, MVT::f64, Legal); 944 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 945 946 // FIXME: Do we need to handle scalar-to-vector here? 947 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 948 949 setOperationAction(ISD::VSELECT, MVT::v2f64, Legal); 950 setOperationAction(ISD::VSELECT, MVT::v2i64, Legal); 951 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); 952 setOperationAction(ISD::VSELECT, MVT::v4i32, Legal); 953 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 954 955 // i8 and i16 vectors are custom , because the source register and source 956 // source memory operand types are not the same width. f32 vectors are 957 // custom since the immediate controlling the insert encodes additional 958 // information. 959 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 960 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 961 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 962 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 963 964 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 965 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 966 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 967 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 968 969 // FIXME: these should be Legal but thats only for the case where 970 // the index is constant. For now custom expand to deal with that. 971 if (Subtarget->is64Bit()) { 972 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 973 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 974 } 975 } 976 977 if (Subtarget->hasSSE2()) { 978 setOperationAction(ISD::SRL, MVT::v8i16, Custom); 979 setOperationAction(ISD::SRL, MVT::v16i8, Custom); 980 981 setOperationAction(ISD::SHL, MVT::v8i16, Custom); 982 setOperationAction(ISD::SHL, MVT::v16i8, Custom); 983 984 setOperationAction(ISD::SRA, MVT::v8i16, Custom); 985 setOperationAction(ISD::SRA, MVT::v16i8, Custom); 986 987 if (Subtarget->hasAVX2()) { 988 setOperationAction(ISD::SRL, MVT::v2i64, Legal); 989 setOperationAction(ISD::SRL, MVT::v4i32, Legal); 990 991 setOperationAction(ISD::SHL, MVT::v2i64, Legal); 992 setOperationAction(ISD::SHL, MVT::v4i32, Legal); 993 994 setOperationAction(ISD::SRA, MVT::v4i32, Legal); 995 } else { 996 setOperationAction(ISD::SRL, MVT::v2i64, Custom); 997 setOperationAction(ISD::SRL, MVT::v4i32, Custom); 998 999 setOperationAction(ISD::SHL, MVT::v2i64, Custom); 1000 setOperationAction(ISD::SHL, MVT::v4i32, Custom); 1001 1002 setOperationAction(ISD::SRA, MVT::v4i32, Custom); 1003 } 1004 } 1005 1006 if (Subtarget->hasSSE42()) 1007 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 1008 1009 if (!TM.Options.UseSoftFloat && Subtarget->hasAVX()) { 1010 addRegisterClass(MVT::v32i8, &X86::VR256RegClass); 1011 addRegisterClass(MVT::v16i16, &X86::VR256RegClass); 1012 addRegisterClass(MVT::v8i32, &X86::VR256RegClass); 1013 addRegisterClass(MVT::v8f32, &X86::VR256RegClass); 1014 addRegisterClass(MVT::v4i64, &X86::VR256RegClass); 1015 addRegisterClass(MVT::v4f64, &X86::VR256RegClass); 1016 1017 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); 1018 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); 1019 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); 1020 1021 setOperationAction(ISD::FADD, MVT::v8f32, Legal); 1022 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); 1023 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); 1024 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); 1025 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); 1026 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); 1027 1028 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 1029 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 1030 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 1031 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 1032 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 1033 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); 1034 1035 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); 1036 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); 1037 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); 1038 1039 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f64, Custom); 1040 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i64, Custom); 1041 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom); 1042 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom); 1043 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i8, Custom); 1044 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i16, Custom); 1045 1046 setOperationAction(ISD::SRL, MVT::v16i16, Custom); 1047 setOperationAction(ISD::SRL, MVT::v32i8, Custom); 1048 1049 setOperationAction(ISD::SHL, MVT::v16i16, Custom); 1050 setOperationAction(ISD::SHL, MVT::v32i8, Custom); 1051 1052 setOperationAction(ISD::SRA, MVT::v16i16, Custom); 1053 setOperationAction(ISD::SRA, MVT::v32i8, Custom); 1054 1055 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); 1056 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); 1057 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); 1058 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); 1059 1060 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); 1061 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); 1062 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); 1063 1064 setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); 1065 setOperationAction(ISD::VSELECT, MVT::v4i64, Legal); 1066 setOperationAction(ISD::VSELECT, MVT::v8i32, Legal); 1067 setOperationAction(ISD::VSELECT, MVT::v8f32, Legal); 1068 1069 if (Subtarget->hasAVX2()) { 1070 setOperationAction(ISD::ADD, MVT::v4i64, Legal); 1071 setOperationAction(ISD::ADD, MVT::v8i32, Legal); 1072 setOperationAction(ISD::ADD, MVT::v16i16, Legal); 1073 setOperationAction(ISD::ADD, MVT::v32i8, Legal); 1074 1075 setOperationAction(ISD::SUB, MVT::v4i64, Legal); 1076 setOperationAction(ISD::SUB, MVT::v8i32, Legal); 1077 setOperationAction(ISD::SUB, MVT::v16i16, Legal); 1078 setOperationAction(ISD::SUB, MVT::v32i8, Legal); 1079 1080 setOperationAction(ISD::MUL, MVT::v4i64, Custom); 1081 setOperationAction(ISD::MUL, MVT::v8i32, Legal); 1082 setOperationAction(ISD::MUL, MVT::v16i16, Legal); 1083 // Don't lower v32i8 because there is no 128-bit byte mul 1084 1085 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); 1086 1087 setOperationAction(ISD::SRL, MVT::v4i64, Legal); 1088 setOperationAction(ISD::SRL, MVT::v8i32, Legal); 1089 1090 setOperationAction(ISD::SHL, MVT::v4i64, Legal); 1091 setOperationAction(ISD::SHL, MVT::v8i32, Legal); 1092 1093 setOperationAction(ISD::SRA, MVT::v8i32, Legal); 1094 } else { 1095 setOperationAction(ISD::ADD, MVT::v4i64, Custom); 1096 setOperationAction(ISD::ADD, MVT::v8i32, Custom); 1097 setOperationAction(ISD::ADD, MVT::v16i16, Custom); 1098 setOperationAction(ISD::ADD, MVT::v32i8, Custom); 1099 1100 setOperationAction(ISD::SUB, MVT::v4i64, Custom); 1101 setOperationAction(ISD::SUB, MVT::v8i32, Custom); 1102 setOperationAction(ISD::SUB, MVT::v16i16, Custom); 1103 setOperationAction(ISD::SUB, MVT::v32i8, Custom); 1104 1105 setOperationAction(ISD::MUL, MVT::v4i64, Custom); 1106 setOperationAction(ISD::MUL, MVT::v8i32, Custom); 1107 setOperationAction(ISD::MUL, MVT::v16i16, Custom); 1108 // Don't lower v32i8 because there is no 128-bit byte mul 1109 1110 setOperationAction(ISD::SRL, MVT::v4i64, Custom); 1111 setOperationAction(ISD::SRL, MVT::v8i32, Custom); 1112 1113 setOperationAction(ISD::SHL, MVT::v4i64, Custom); 1114 setOperationAction(ISD::SHL, MVT::v8i32, Custom); 1115 1116 setOperationAction(ISD::SRA, MVT::v8i32, Custom); 1117 } 1118 1119 // Custom lower several nodes for 256-bit types. 1120 for (int i = MVT::FIRST_VECTOR_VALUETYPE; 1121 i <= MVT::LAST_VECTOR_VALUETYPE; ++i) { 1122 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 1123 EVT VT = SVT; 1124 1125 // Extract subvector is special because the value type 1126 // (result) is 128-bit but the source is 256-bit wide. 1127 if (VT.is128BitVector()) 1128 setOperationAction(ISD::EXTRACT_SUBVECTOR, SVT, Custom); 1129 1130 // Do not attempt to custom lower other non-256-bit vectors 1131 if (!VT.is256BitVector()) 1132 continue; 1133 1134 setOperationAction(ISD::BUILD_VECTOR, SVT, Custom); 1135 setOperationAction(ISD::VECTOR_SHUFFLE, SVT, Custom); 1136 setOperationAction(ISD::INSERT_VECTOR_ELT, SVT, Custom); 1137 setOperationAction(ISD::EXTRACT_VECTOR_ELT, SVT, Custom); 1138 setOperationAction(ISD::SCALAR_TO_VECTOR, SVT, Custom); 1139 setOperationAction(ISD::INSERT_SUBVECTOR, SVT, Custom); 1140 } 1141 1142 // Promote v32i8, v16i16, v8i32 select, and, or, xor to v4i64. 1143 for (int i = MVT::v32i8; i != MVT::v4i64; ++i) { 1144 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 1145 EVT VT = SVT; 1146 1147 // Do not attempt to promote non-256-bit vectors 1148 if (!VT.is256BitVector()) 1149 continue; 1150 1151 setOperationAction(ISD::AND, SVT, Promote); 1152 AddPromotedToType (ISD::AND, SVT, MVT::v4i64); 1153 setOperationAction(ISD::OR, SVT, Promote); 1154 AddPromotedToType (ISD::OR, SVT, MVT::v4i64); 1155 setOperationAction(ISD::XOR, SVT, Promote); 1156 AddPromotedToType (ISD::XOR, SVT, MVT::v4i64); 1157 setOperationAction(ISD::LOAD, SVT, Promote); 1158 AddPromotedToType (ISD::LOAD, SVT, MVT::v4i64); 1159 setOperationAction(ISD::SELECT, SVT, Promote); 1160 AddPromotedToType (ISD::SELECT, SVT, MVT::v4i64); 1161 } 1162 } 1163 1164 // SIGN_EXTEND_INREGs are evaluated by the extend type. Handle the expansion 1165 // of this type with custom code. 1166 for (int VT = MVT::FIRST_VECTOR_VALUETYPE; 1167 VT != MVT::LAST_VECTOR_VALUETYPE; VT++) { 1168 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT, 1169 Custom); 1170 } 1171 1172 // We want to custom lower some of our intrinsics. 1173 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 1174 1175 1176 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't 1177 // handle type legalization for these operations here. 1178 // 1179 // FIXME: We really should do custom legalization for addition and 1180 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better 1181 // than generic legalization for 64-bit multiplication-with-overflow, though. 1182 for (unsigned i = 0, e = 3+Subtarget->is64Bit(); i != e; ++i) { 1183 // Add/Sub/Mul with overflow operations are custom lowered. 1184 MVT VT = IntVTs[i]; 1185 setOperationAction(ISD::SADDO, VT, Custom); 1186 setOperationAction(ISD::UADDO, VT, Custom); 1187 setOperationAction(ISD::SSUBO, VT, Custom); 1188 setOperationAction(ISD::USUBO, VT, Custom); 1189 setOperationAction(ISD::SMULO, VT, Custom); 1190 setOperationAction(ISD::UMULO, VT, Custom); 1191 } 1192 1193 // There are no 8-bit 3-address imul/mul instructions 1194 setOperationAction(ISD::SMULO, MVT::i8, Expand); 1195 setOperationAction(ISD::UMULO, MVT::i8, Expand); 1196 1197 if (!Subtarget->is64Bit()) { 1198 // These libcalls are not available in 32-bit. 1199 setLibcallName(RTLIB::SHL_I128, 0); 1200 setLibcallName(RTLIB::SRL_I128, 0); 1201 setLibcallName(RTLIB::SRA_I128, 0); 1202 } 1203 1204 // We have target-specific dag combine patterns for the following nodes: 1205 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 1206 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 1207 setTargetDAGCombine(ISD::VSELECT); 1208 setTargetDAGCombine(ISD::SELECT); 1209 setTargetDAGCombine(ISD::SHL); 1210 setTargetDAGCombine(ISD::SRA); 1211 setTargetDAGCombine(ISD::SRL); 1212 setTargetDAGCombine(ISD::OR); 1213 setTargetDAGCombine(ISD::AND); 1214 setTargetDAGCombine(ISD::ADD); 1215 setTargetDAGCombine(ISD::FADD); 1216 setTargetDAGCombine(ISD::FSUB); 1217 setTargetDAGCombine(ISD::SUB); 1218 setTargetDAGCombine(ISD::LOAD); 1219 setTargetDAGCombine(ISD::STORE); 1220 setTargetDAGCombine(ISD::ZERO_EXTEND); 1221 setTargetDAGCombine(ISD::ANY_EXTEND); 1222 setTargetDAGCombine(ISD::SIGN_EXTEND); 1223 setTargetDAGCombine(ISD::TRUNCATE); 1224 setTargetDAGCombine(ISD::UINT_TO_FP); 1225 setTargetDAGCombine(ISD::SINT_TO_FP); 1226 setTargetDAGCombine(ISD::SETCC); 1227 setTargetDAGCombine(ISD::FP_TO_SINT); 1228 if (Subtarget->is64Bit()) 1229 setTargetDAGCombine(ISD::MUL); 1230 if (Subtarget->hasBMI()) 1231 setTargetDAGCombine(ISD::XOR); 1232 1233 computeRegisterProperties(); 1234 1235 // On Darwin, -Os means optimize for size without hurting performance, 1236 // do not reduce the limit. 1237 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 1238 maxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 16 : 8; 1239 maxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores 1240 maxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 8 : 4; 1241 maxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores 1242 maxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 8 : 4; 1243 setPrefLoopAlignment(4); // 2^4 bytes. 1244 benefitFromCodePlacementOpt = true; 1245 1246 // Predictable cmov don't hurt on atom because it's in-order. 1247 predictableSelectIsExpensive = !Subtarget->isAtom(); 1248 1249 setPrefFunctionAlignment(4); // 2^4 bytes. 1250} 1251 1252 1253EVT X86TargetLowering::getSetCCResultType(EVT VT) const { 1254 if (!VT.isVector()) return MVT::i8; 1255 return VT.changeVectorElementTypeToInteger(); 1256} 1257 1258 1259/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 1260/// the desired ByVal argument alignment. 1261static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) { 1262 if (MaxAlign == 16) 1263 return; 1264 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1265 if (VTy->getBitWidth() == 128) 1266 MaxAlign = 16; 1267 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1268 unsigned EltAlign = 0; 1269 getMaxByValAlign(ATy->getElementType(), EltAlign); 1270 if (EltAlign > MaxAlign) 1271 MaxAlign = EltAlign; 1272 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1273 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 1274 unsigned EltAlign = 0; 1275 getMaxByValAlign(STy->getElementType(i), EltAlign); 1276 if (EltAlign > MaxAlign) 1277 MaxAlign = EltAlign; 1278 if (MaxAlign == 16) 1279 break; 1280 } 1281 } 1282} 1283 1284/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1285/// function arguments in the caller parameter area. For X86, aggregates 1286/// that contain SSE vectors are placed at 16-byte boundaries while the rest 1287/// are at 4-byte boundaries. 1288unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const { 1289 if (Subtarget->is64Bit()) { 1290 // Max of 8 and alignment of type. 1291 unsigned TyAlign = TD->getABITypeAlignment(Ty); 1292 if (TyAlign > 8) 1293 return TyAlign; 1294 return 8; 1295 } 1296 1297 unsigned Align = 4; 1298 if (Subtarget->hasSSE1()) 1299 getMaxByValAlign(Ty, Align); 1300 return Align; 1301} 1302 1303/// getOptimalMemOpType - Returns the target specific optimal type for load 1304/// and store operations as a result of memset, memcpy, and memmove 1305/// lowering. If DstAlign is zero that means it's safe to destination 1306/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 1307/// means there isn't a need to check it against alignment requirement, 1308/// probably because the source does not need to be loaded. If 1309/// 'IsZeroVal' is true, that means it's safe to return a 1310/// non-scalar-integer type, e.g. empty string source, constant, or loaded 1311/// from memory. 'MemcpyStrSrc' indicates whether the memcpy source is 1312/// constant so it does not need to be loaded. 1313/// It returns EVT::Other if the type should be determined using generic 1314/// target-independent logic. 1315EVT 1316X86TargetLowering::getOptimalMemOpType(uint64_t Size, 1317 unsigned DstAlign, unsigned SrcAlign, 1318 bool IsZeroVal, 1319 bool MemcpyStrSrc, 1320 MachineFunction &MF) const { 1321 // FIXME: This turns off use of xmm stores for memset/memcpy on targets like 1322 // linux. This is because the stack realignment code can't handle certain 1323 // cases like PR2962. This should be removed when PR2962 is fixed. 1324 const Function *F = MF.getFunction(); 1325 if (IsZeroVal && 1326 !F->hasFnAttr(Attribute::NoImplicitFloat)) { 1327 if (Size >= 16 && 1328 (Subtarget->isUnalignedMemAccessFast() || 1329 ((DstAlign == 0 || DstAlign >= 16) && 1330 (SrcAlign == 0 || SrcAlign >= 16))) && 1331 Subtarget->getStackAlignment() >= 16) { 1332 if (Subtarget->getStackAlignment() >= 32) { 1333 if (Subtarget->hasAVX2()) 1334 return MVT::v8i32; 1335 if (Subtarget->hasAVX()) 1336 return MVT::v8f32; 1337 } 1338 if (Subtarget->hasSSE2()) 1339 return MVT::v4i32; 1340 if (Subtarget->hasSSE1()) 1341 return MVT::v4f32; 1342 } else if (!MemcpyStrSrc && Size >= 8 && 1343 !Subtarget->is64Bit() && 1344 Subtarget->getStackAlignment() >= 8 && 1345 Subtarget->hasSSE2()) { 1346 // Do not use f64 to lower memcpy if source is string constant. It's 1347 // better to use i32 to avoid the loads. 1348 return MVT::f64; 1349 } 1350 } 1351 if (Subtarget->is64Bit() && Size >= 8) 1352 return MVT::i64; 1353 return MVT::i32; 1354} 1355 1356/// getJumpTableEncoding - Return the entry encoding for a jump table in the 1357/// current function. The returned value is a member of the 1358/// MachineJumpTableInfo::JTEntryKind enum. 1359unsigned X86TargetLowering::getJumpTableEncoding() const { 1360 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF 1361 // symbol. 1362 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1363 Subtarget->isPICStyleGOT()) 1364 return MachineJumpTableInfo::EK_Custom32; 1365 1366 // Otherwise, use the normal jump table encoding heuristics. 1367 return TargetLowering::getJumpTableEncoding(); 1368} 1369 1370const MCExpr * 1371X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, 1372 const MachineBasicBlock *MBB, 1373 unsigned uid,MCContext &Ctx) const{ 1374 assert(getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1375 Subtarget->isPICStyleGOT()); 1376 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF 1377 // entries. 1378 return MCSymbolRefExpr::Create(MBB->getSymbol(), 1379 MCSymbolRefExpr::VK_GOTOFF, Ctx); 1380} 1381 1382/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 1383/// jumptable. 1384SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 1385 SelectionDAG &DAG) const { 1386 if (!Subtarget->is64Bit()) 1387 // This doesn't have DebugLoc associated with it, but is not really the 1388 // same as a Register. 1389 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy()); 1390 return Table; 1391} 1392 1393/// getPICJumpTableRelocBaseExpr - This returns the relocation base for the 1394/// given PIC jumptable, the same as getPICJumpTableRelocBase, but as an 1395/// MCExpr. 1396const MCExpr *X86TargetLowering:: 1397getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, 1398 MCContext &Ctx) const { 1399 // X86-64 uses RIP relative addressing based on the jump table label. 1400 if (Subtarget->isPICStyleRIPRel()) 1401 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 1402 1403 // Otherwise, the reference is relative to the PIC base. 1404 return MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx); 1405} 1406 1407// FIXME: Why this routine is here? Move to RegInfo! 1408std::pair<const TargetRegisterClass*, uint8_t> 1409X86TargetLowering::findRepresentativeClass(EVT VT) const{ 1410 const TargetRegisterClass *RRC = 0; 1411 uint8_t Cost = 1; 1412 switch (VT.getSimpleVT().SimpleTy) { 1413 default: 1414 return TargetLowering::findRepresentativeClass(VT); 1415 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: 1416 RRC = Subtarget->is64Bit() ? 1417 (const TargetRegisterClass*)&X86::GR64RegClass : 1418 (const TargetRegisterClass*)&X86::GR32RegClass; 1419 break; 1420 case MVT::x86mmx: 1421 RRC = &X86::VR64RegClass; 1422 break; 1423 case MVT::f32: case MVT::f64: 1424 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: 1425 case MVT::v4f32: case MVT::v2f64: 1426 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: 1427 case MVT::v4f64: 1428 RRC = &X86::VR128RegClass; 1429 break; 1430 } 1431 return std::make_pair(RRC, Cost); 1432} 1433 1434bool X86TargetLowering::getStackCookieLocation(unsigned &AddressSpace, 1435 unsigned &Offset) const { 1436 if (!Subtarget->isTargetLinux()) 1437 return false; 1438 1439 if (Subtarget->is64Bit()) { 1440 // %fs:0x28, unless we're using a Kernel code model, in which case it's %gs: 1441 Offset = 0x28; 1442 if (getTargetMachine().getCodeModel() == CodeModel::Kernel) 1443 AddressSpace = 256; 1444 else 1445 AddressSpace = 257; 1446 } else { 1447 // %gs:0x14 on i386 1448 Offset = 0x14; 1449 AddressSpace = 256; 1450 } 1451 return true; 1452} 1453 1454 1455//===----------------------------------------------------------------------===// 1456// Return Value Calling Convention Implementation 1457//===----------------------------------------------------------------------===// 1458 1459#include "X86GenCallingConv.inc" 1460 1461bool 1462X86TargetLowering::CanLowerReturn(CallingConv::ID CallConv, 1463 MachineFunction &MF, bool isVarArg, 1464 const SmallVectorImpl<ISD::OutputArg> &Outs, 1465 LLVMContext &Context) const { 1466 SmallVector<CCValAssign, 16> RVLocs; 1467 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1468 RVLocs, Context); 1469 return CCInfo.CheckReturn(Outs, RetCC_X86); 1470} 1471 1472SDValue 1473X86TargetLowering::LowerReturn(SDValue Chain, 1474 CallingConv::ID CallConv, bool isVarArg, 1475 const SmallVectorImpl<ISD::OutputArg> &Outs, 1476 const SmallVectorImpl<SDValue> &OutVals, 1477 DebugLoc dl, SelectionDAG &DAG) const { 1478 MachineFunction &MF = DAG.getMachineFunction(); 1479 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1480 1481 SmallVector<CCValAssign, 16> RVLocs; 1482 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1483 RVLocs, *DAG.getContext()); 1484 CCInfo.AnalyzeReturn(Outs, RetCC_X86); 1485 1486 // Add the regs to the liveout set for the function. 1487 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 1488 for (unsigned i = 0; i != RVLocs.size(); ++i) 1489 if (RVLocs[i].isRegLoc() && !MRI.isLiveOut(RVLocs[i].getLocReg())) 1490 MRI.addLiveOut(RVLocs[i].getLocReg()); 1491 1492 SDValue Flag; 1493 1494 SmallVector<SDValue, 6> RetOps; 1495 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 1496 // Operand #1 = Bytes To Pop 1497 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), 1498 MVT::i16)); 1499 1500 // Copy the result values into the output registers. 1501 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1502 CCValAssign &VA = RVLocs[i]; 1503 assert(VA.isRegLoc() && "Can only return in registers!"); 1504 SDValue ValToCopy = OutVals[i]; 1505 EVT ValVT = ValToCopy.getValueType(); 1506 1507 // If this is x86-64, and we disabled SSE, we can't return FP values, 1508 // or SSE or MMX vectors. 1509 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || 1510 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) && 1511 (Subtarget->is64Bit() && !Subtarget->hasSSE1())) { 1512 report_fatal_error("SSE register return with SSE disabled"); 1513 } 1514 // Likewise we can't return F64 values with SSE1 only. gcc does so, but 1515 // llvm-gcc has never done it right and no one has noticed, so this 1516 // should be OK for now. 1517 if (ValVT == MVT::f64 && 1518 (Subtarget->is64Bit() && !Subtarget->hasSSE2())) 1519 report_fatal_error("SSE2 register return with SSE2 disabled"); 1520 1521 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 1522 // the RET instruction and handled by the FP Stackifier. 1523 if (VA.getLocReg() == X86::ST0 || 1524 VA.getLocReg() == X86::ST1) { 1525 // If this is a copy from an xmm register to ST(0), use an FPExtend to 1526 // change the value to the FP stack register class. 1527 if (isScalarFPTypeInSSEReg(VA.getValVT())) 1528 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); 1529 RetOps.push_back(ValToCopy); 1530 // Don't emit a copytoreg. 1531 continue; 1532 } 1533 1534 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64 1535 // which is returned in RAX / RDX. 1536 if (Subtarget->is64Bit()) { 1537 if (ValVT == MVT::x86mmx) { 1538 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { 1539 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); 1540 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, 1541 ValToCopy); 1542 // If we don't have SSE2 available, convert to v4f32 so the generated 1543 // register is legal. 1544 if (!Subtarget->hasSSE2()) 1545 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); 1546 } 1547 } 1548 } 1549 1550 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag); 1551 Flag = Chain.getValue(1); 1552 } 1553 1554 // The x86-64 ABI for returning structs by value requires that we copy 1555 // the sret argument into %rax for the return. We saved the argument into 1556 // a virtual register in the entry block, so now we copy the value out 1557 // and into %rax. 1558 if (Subtarget->is64Bit() && 1559 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1560 MachineFunction &MF = DAG.getMachineFunction(); 1561 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1562 unsigned Reg = FuncInfo->getSRetReturnReg(); 1563 assert(Reg && 1564 "SRetReturnReg should have been set in LowerFormalArguments()."); 1565 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy()); 1566 1567 Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag); 1568 Flag = Chain.getValue(1); 1569 1570 // RAX now acts like a return value. 1571 MRI.addLiveOut(X86::RAX); 1572 } 1573 1574 RetOps[0] = Chain; // Update chain. 1575 1576 // Add the flag if we have it. 1577 if (Flag.getNode()) 1578 RetOps.push_back(Flag); 1579 1580 return DAG.getNode(X86ISD::RET_FLAG, dl, 1581 MVT::Other, &RetOps[0], RetOps.size()); 1582} 1583 1584bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { 1585 if (N->getNumValues() != 1) 1586 return false; 1587 if (!N->hasNUsesOfValue(1, 0)) 1588 return false; 1589 1590 SDValue TCChain = Chain; 1591 SDNode *Copy = *N->use_begin(); 1592 if (Copy->getOpcode() == ISD::CopyToReg) { 1593 // If the copy has a glue operand, we conservatively assume it isn't safe to 1594 // perform a tail call. 1595 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) 1596 return false; 1597 TCChain = Copy->getOperand(0); 1598 } else if (Copy->getOpcode() != ISD::FP_EXTEND) 1599 return false; 1600 1601 bool HasRet = false; 1602 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); 1603 UI != UE; ++UI) { 1604 if (UI->getOpcode() != X86ISD::RET_FLAG) 1605 return false; 1606 HasRet = true; 1607 } 1608 1609 if (!HasRet) 1610 return false; 1611 1612 Chain = TCChain; 1613 return true; 1614} 1615 1616EVT 1617X86TargetLowering::getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT, 1618 ISD::NodeType ExtendKind) const { 1619 MVT ReturnMVT; 1620 // TODO: Is this also valid on 32-bit? 1621 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) 1622 ReturnMVT = MVT::i8; 1623 else 1624 ReturnMVT = MVT::i32; 1625 1626 EVT MinVT = getRegisterType(Context, ReturnMVT); 1627 return VT.bitsLT(MinVT) ? MinVT : VT; 1628} 1629 1630/// LowerCallResult - Lower the result values of a call into the 1631/// appropriate copies out of appropriate physical registers. 1632/// 1633SDValue 1634X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag, 1635 CallingConv::ID CallConv, bool isVarArg, 1636 const SmallVectorImpl<ISD::InputArg> &Ins, 1637 DebugLoc dl, SelectionDAG &DAG, 1638 SmallVectorImpl<SDValue> &InVals) const { 1639 1640 // Assign locations to each value returned by this call. 1641 SmallVector<CCValAssign, 16> RVLocs; 1642 bool Is64Bit = Subtarget->is64Bit(); 1643 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 1644 getTargetMachine(), RVLocs, *DAG.getContext()); 1645 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 1646 1647 // Copy all of the result registers out of their specified physreg. 1648 for (unsigned i = 0; i != RVLocs.size(); ++i) { 1649 CCValAssign &VA = RVLocs[i]; 1650 EVT CopyVT = VA.getValVT(); 1651 1652 // If this is x86-64, and we disabled SSE, we can't return FP values 1653 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) && 1654 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) { 1655 report_fatal_error("SSE register return with SSE disabled"); 1656 } 1657 1658 SDValue Val; 1659 1660 // If this is a call to a function that returns an fp value on the floating 1661 // point stack, we must guarantee the the value is popped from the stack, so 1662 // a CopyFromReg is not good enough - the copy instruction may be eliminated 1663 // if the return value is not used. We use the FpPOP_RETVAL instruction 1664 // instead. 1665 if (VA.getLocReg() == X86::ST0 || VA.getLocReg() == X86::ST1) { 1666 // If we prefer to use the value in xmm registers, copy it out as f80 and 1667 // use a truncate to move it from fp stack reg to xmm reg. 1668 if (isScalarFPTypeInSSEReg(VA.getValVT())) CopyVT = MVT::f80; 1669 SDValue Ops[] = { Chain, InFlag }; 1670 Chain = SDValue(DAG.getMachineNode(X86::FpPOP_RETVAL, dl, CopyVT, 1671 MVT::Other, MVT::Glue, Ops, 2), 1); 1672 Val = Chain.getValue(0); 1673 1674 // Round the f80 to the right size, which also moves it to the appropriate 1675 // xmm register. 1676 if (CopyVT != VA.getValVT()) 1677 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, 1678 // This truncation won't change the value. 1679 DAG.getIntPtrConstant(1)); 1680 } else { 1681 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 1682 CopyVT, InFlag).getValue(1); 1683 Val = Chain.getValue(0); 1684 } 1685 InFlag = Chain.getValue(2); 1686 InVals.push_back(Val); 1687 } 1688 1689 return Chain; 1690} 1691 1692 1693//===----------------------------------------------------------------------===// 1694// C & StdCall & Fast Calling Convention implementation 1695//===----------------------------------------------------------------------===// 1696// StdCall calling convention seems to be standard for many Windows' API 1697// routines and around. It differs from C calling convention just a little: 1698// callee should clean up the stack, not caller. Symbols should be also 1699// decorated in some fancy way :) It doesn't support any vector arguments. 1700// For info on fast calling convention see Fast Calling Convention (tail call) 1701// implementation LowerX86_32FastCCCallTo. 1702 1703/// CallIsStructReturn - Determines whether a call uses struct return 1704/// semantics. 1705static bool CallIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) { 1706 if (Outs.empty()) 1707 return false; 1708 1709 return Outs[0].Flags.isSRet(); 1710} 1711 1712/// ArgsAreStructReturn - Determines whether a function uses struct 1713/// return semantics. 1714static bool 1715ArgsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) { 1716 if (Ins.empty()) 1717 return false; 1718 1719 return Ins[0].Flags.isSRet(); 1720} 1721 1722/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1723/// by "Src" to address "Dst" with size and alignment information specified by 1724/// the specific parameter attribute. The copy will be passed as a byval 1725/// function parameter. 1726static SDValue 1727CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 1728 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, 1729 DebugLoc dl) { 1730 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1731 1732 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 1733 /*isVolatile*/false, /*AlwaysInline=*/true, 1734 MachinePointerInfo(), MachinePointerInfo()); 1735} 1736 1737/// IsTailCallConvention - Return true if the calling convention is one that 1738/// supports tail call optimization. 1739static bool IsTailCallConvention(CallingConv::ID CC) { 1740 return (CC == CallingConv::Fast || CC == CallingConv::GHC); 1741} 1742 1743bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 1744 if (!CI->isTailCall() || getTargetMachine().Options.DisableTailCalls) 1745 return false; 1746 1747 CallSite CS(CI); 1748 CallingConv::ID CalleeCC = CS.getCallingConv(); 1749 if (!IsTailCallConvention(CalleeCC) && CalleeCC != CallingConv::C) 1750 return false; 1751 1752 return true; 1753} 1754 1755/// FuncIsMadeTailCallSafe - Return true if the function is being made into 1756/// a tailcall target by changing its ABI. 1757static bool FuncIsMadeTailCallSafe(CallingConv::ID CC, 1758 bool GuaranteedTailCallOpt) { 1759 return GuaranteedTailCallOpt && IsTailCallConvention(CC); 1760} 1761 1762SDValue 1763X86TargetLowering::LowerMemArgument(SDValue Chain, 1764 CallingConv::ID CallConv, 1765 const SmallVectorImpl<ISD::InputArg> &Ins, 1766 DebugLoc dl, SelectionDAG &DAG, 1767 const CCValAssign &VA, 1768 MachineFrameInfo *MFI, 1769 unsigned i) const { 1770 // Create the nodes corresponding to a load from this parameter slot. 1771 ISD::ArgFlagsTy Flags = Ins[i].Flags; 1772 bool AlwaysUseMutable = FuncIsMadeTailCallSafe(CallConv, 1773 getTargetMachine().Options.GuaranteedTailCallOpt); 1774 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1775 EVT ValVT; 1776 1777 // If value is passed by pointer we have address passed instead of the value 1778 // itself. 1779 if (VA.getLocInfo() == CCValAssign::Indirect) 1780 ValVT = VA.getLocVT(); 1781 else 1782 ValVT = VA.getValVT(); 1783 1784 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1785 // changed with more analysis. 1786 // In case of tail call optimization mark all arguments mutable. Since they 1787 // could be overwritten by lowering of arguments in case of a tail call. 1788 if (Flags.isByVal()) { 1789 unsigned Bytes = Flags.getByValSize(); 1790 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects. 1791 int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable); 1792 return DAG.getFrameIndex(FI, getPointerTy()); 1793 } else { 1794 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8, 1795 VA.getLocMemOffset(), isImmutable); 1796 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 1797 return DAG.getLoad(ValVT, dl, Chain, FIN, 1798 MachinePointerInfo::getFixedStack(FI), 1799 false, false, false, 0); 1800 } 1801} 1802 1803SDValue 1804X86TargetLowering::LowerFormalArguments(SDValue Chain, 1805 CallingConv::ID CallConv, 1806 bool isVarArg, 1807 const SmallVectorImpl<ISD::InputArg> &Ins, 1808 DebugLoc dl, 1809 SelectionDAG &DAG, 1810 SmallVectorImpl<SDValue> &InVals) 1811 const { 1812 MachineFunction &MF = DAG.getMachineFunction(); 1813 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1814 1815 const Function* Fn = MF.getFunction(); 1816 if (Fn->hasExternalLinkage() && 1817 Subtarget->isTargetCygMing() && 1818 Fn->getName() == "main") 1819 FuncInfo->setForceFramePointer(true); 1820 1821 MachineFrameInfo *MFI = MF.getFrameInfo(); 1822 bool Is64Bit = Subtarget->is64Bit(); 1823 bool IsWindows = Subtarget->isTargetWindows(); 1824 bool IsWin64 = Subtarget->isTargetWin64(); 1825 1826 assert(!(isVarArg && IsTailCallConvention(CallConv)) && 1827 "Var args not supported with calling convention fastcc or ghc"); 1828 1829 // Assign locations to all of the incoming arguments. 1830 SmallVector<CCValAssign, 16> ArgLocs; 1831 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 1832 ArgLocs, *DAG.getContext()); 1833 1834 // Allocate shadow area for Win64 1835 if (IsWin64) { 1836 CCInfo.AllocateStack(32, 8); 1837 } 1838 1839 CCInfo.AnalyzeFormalArguments(Ins, CC_X86); 1840 1841 unsigned LastVal = ~0U; 1842 SDValue ArgValue; 1843 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1844 CCValAssign &VA = ArgLocs[i]; 1845 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1846 // places. 1847 assert(VA.getValNo() != LastVal && 1848 "Don't support value assigned to multiple locs yet"); 1849 (void)LastVal; 1850 LastVal = VA.getValNo(); 1851 1852 if (VA.isRegLoc()) { 1853 EVT RegVT = VA.getLocVT(); 1854 const TargetRegisterClass *RC; 1855 if (RegVT == MVT::i32) 1856 RC = &X86::GR32RegClass; 1857 else if (Is64Bit && RegVT == MVT::i64) 1858 RC = &X86::GR64RegClass; 1859 else if (RegVT == MVT::f32) 1860 RC = &X86::FR32RegClass; 1861 else if (RegVT == MVT::f64) 1862 RC = &X86::FR64RegClass; 1863 else if (RegVT.isVector() && RegVT.getSizeInBits() == 256) 1864 RC = &X86::VR256RegClass; 1865 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1866 RC = &X86::VR128RegClass; 1867 else if (RegVT == MVT::x86mmx) 1868 RC = &X86::VR64RegClass; 1869 else 1870 llvm_unreachable("Unknown argument type!"); 1871 1872 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 1873 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 1874 1875 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1876 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1877 // right size. 1878 if (VA.getLocInfo() == CCValAssign::SExt) 1879 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 1880 DAG.getValueType(VA.getValVT())); 1881 else if (VA.getLocInfo() == CCValAssign::ZExt) 1882 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 1883 DAG.getValueType(VA.getValVT())); 1884 else if (VA.getLocInfo() == CCValAssign::BCvt) 1885 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); 1886 1887 if (VA.isExtInLoc()) { 1888 // Handle MMX values passed in XMM regs. 1889 if (RegVT.isVector()) { 1890 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), 1891 ArgValue); 1892 } else 1893 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 1894 } 1895 } else { 1896 assert(VA.isMemLoc()); 1897 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i); 1898 } 1899 1900 // If value is passed via pointer - do a load. 1901 if (VA.getLocInfo() == CCValAssign::Indirect) 1902 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, 1903 MachinePointerInfo(), false, false, false, 0); 1904 1905 InVals.push_back(ArgValue); 1906 } 1907 1908 // The x86-64 ABI for returning structs by value requires that we copy 1909 // the sret argument into %rax for the return. Save the argument into 1910 // a virtual register so that we can access it from the return points. 1911 if (Is64Bit && MF.getFunction()->hasStructRetAttr()) { 1912 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1913 unsigned Reg = FuncInfo->getSRetReturnReg(); 1914 if (!Reg) { 1915 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1916 FuncInfo->setSRetReturnReg(Reg); 1917 } 1918 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]); 1919 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); 1920 } 1921 1922 unsigned StackSize = CCInfo.getNextStackOffset(); 1923 // Align stack specially for tail calls. 1924 if (FuncIsMadeTailCallSafe(CallConv, 1925 MF.getTarget().Options.GuaranteedTailCallOpt)) 1926 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1927 1928 // If the function takes variable number of arguments, make a frame index for 1929 // the start of the first vararg value... for expansion of llvm.va_start. 1930 if (isVarArg) { 1931 if (Is64Bit || (CallConv != CallingConv::X86_FastCall && 1932 CallConv != CallingConv::X86_ThisCall)) { 1933 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize,true)); 1934 } 1935 if (Is64Bit) { 1936 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1937 1938 // FIXME: We should really autogenerate these arrays 1939 static const uint16_t GPR64ArgRegsWin64[] = { 1940 X86::RCX, X86::RDX, X86::R8, X86::R9 1941 }; 1942 static const uint16_t GPR64ArgRegs64Bit[] = { 1943 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1944 }; 1945 static const uint16_t XMMArgRegs64Bit[] = { 1946 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1947 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1948 }; 1949 const uint16_t *GPR64ArgRegs; 1950 unsigned NumXMMRegs = 0; 1951 1952 if (IsWin64) { 1953 // The XMM registers which might contain var arg parameters are shadowed 1954 // in their paired GPR. So we only need to save the GPR to their home 1955 // slots. 1956 TotalNumIntRegs = 4; 1957 GPR64ArgRegs = GPR64ArgRegsWin64; 1958 } else { 1959 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1960 GPR64ArgRegs = GPR64ArgRegs64Bit; 1961 1962 NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs64Bit, 1963 TotalNumXMMRegs); 1964 } 1965 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1966 TotalNumIntRegs); 1967 1968 bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat); 1969 assert(!(NumXMMRegs && !Subtarget->hasSSE1()) && 1970 "SSE register cannot be used when SSE is disabled!"); 1971 assert(!(NumXMMRegs && MF.getTarget().Options.UseSoftFloat && 1972 NoImplicitFloatOps) && 1973 "SSE register cannot be used when SSE is disabled!"); 1974 if (MF.getTarget().Options.UseSoftFloat || NoImplicitFloatOps || 1975 !Subtarget->hasSSE1()) 1976 // Kernel mode asks for SSE to be disabled, so don't push them 1977 // on the stack. 1978 TotalNumXMMRegs = 0; 1979 1980 if (IsWin64) { 1981 const TargetFrameLowering &TFI = *getTargetMachine().getFrameLowering(); 1982 // Get to the caller-allocated home save location. Add 8 to account 1983 // for the return address. 1984 int HomeOffset = TFI.getOffsetOfLocalArea() + 8; 1985 FuncInfo->setRegSaveFrameIndex( 1986 MFI->CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false)); 1987 // Fixup to set vararg frame on shadow area (4 x i64). 1988 if (NumIntRegs < 4) 1989 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex()); 1990 } else { 1991 // For X86-64, if there are vararg parameters that are passed via 1992 // registers, then we must store them to their spots on the stack so 1993 // they may be loaded by deferencing the result of va_next. 1994 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8); 1995 FuncInfo->setVarArgsFPOffset(TotalNumIntRegs * 8 + NumXMMRegs * 16); 1996 FuncInfo->setRegSaveFrameIndex( 1997 MFI->CreateStackObject(TotalNumIntRegs * 8 + TotalNumXMMRegs * 16, 16, 1998 false)); 1999 } 2000 2001 // Store the integer parameter registers. 2002 SmallVector<SDValue, 8> MemOps; 2003 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), 2004 getPointerTy()); 2005 unsigned Offset = FuncInfo->getVarArgsGPOffset(); 2006 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 2007 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN, 2008 DAG.getIntPtrConstant(Offset)); 2009 unsigned VReg = MF.addLiveIn(GPR64ArgRegs[NumIntRegs], 2010 &X86::GR64RegClass); 2011 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 2012 SDValue Store = 2013 DAG.getStore(Val.getValue(1), dl, Val, FIN, 2014 MachinePointerInfo::getFixedStack( 2015 FuncInfo->getRegSaveFrameIndex(), Offset), 2016 false, false, 0); 2017 MemOps.push_back(Store); 2018 Offset += 8; 2019 } 2020 2021 if (TotalNumXMMRegs != 0 && NumXMMRegs != TotalNumXMMRegs) { 2022 // Now store the XMM (fp + vector) parameter registers. 2023 SmallVector<SDValue, 11> SaveXMMOps; 2024 SaveXMMOps.push_back(Chain); 2025 2026 unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass); 2027 SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8); 2028 SaveXMMOps.push_back(ALVal); 2029 2030 SaveXMMOps.push_back(DAG.getIntPtrConstant( 2031 FuncInfo->getRegSaveFrameIndex())); 2032 SaveXMMOps.push_back(DAG.getIntPtrConstant( 2033 FuncInfo->getVarArgsFPOffset())); 2034 2035 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 2036 unsigned VReg = MF.addLiveIn(XMMArgRegs64Bit[NumXMMRegs], 2037 &X86::VR128RegClass); 2038 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32); 2039 SaveXMMOps.push_back(Val); 2040 } 2041 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, 2042 MVT::Other, 2043 &SaveXMMOps[0], SaveXMMOps.size())); 2044 } 2045 2046 if (!MemOps.empty()) 2047 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 2048 &MemOps[0], MemOps.size()); 2049 } 2050 } 2051 2052 // Some CCs need callee pop. 2053 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, 2054 MF.getTarget().Options.GuaranteedTailCallOpt)) { 2055 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything. 2056 } else { 2057 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing. 2058 // If this is an sret function, the return should pop the hidden pointer. 2059 if (!Is64Bit && !IsTailCallConvention(CallConv) && !IsWindows && 2060 ArgsAreStructReturn(Ins)) 2061 FuncInfo->setBytesToPopOnReturn(4); 2062 } 2063 2064 if (!Is64Bit) { 2065 // RegSaveFrameIndex is X86-64 only. 2066 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); 2067 if (CallConv == CallingConv::X86_FastCall || 2068 CallConv == CallingConv::X86_ThisCall) 2069 // fastcc functions can't have varargs. 2070 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA); 2071 } 2072 2073 FuncInfo->setArgumentStackSize(StackSize); 2074 2075 return Chain; 2076} 2077 2078SDValue 2079X86TargetLowering::LowerMemOpCallTo(SDValue Chain, 2080 SDValue StackPtr, SDValue Arg, 2081 DebugLoc dl, SelectionDAG &DAG, 2082 const CCValAssign &VA, 2083 ISD::ArgFlagsTy Flags) const { 2084 unsigned LocMemOffset = VA.getLocMemOffset(); 2085 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 2086 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); 2087 if (Flags.isByVal()) 2088 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); 2089 2090 return DAG.getStore(Chain, dl, Arg, PtrOff, 2091 MachinePointerInfo::getStack(LocMemOffset), 2092 false, false, 0); 2093} 2094 2095/// EmitTailCallLoadRetAddr - Emit a load of return address if tail call 2096/// optimization is performed and it is required. 2097SDValue 2098X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 2099 SDValue &OutRetAddr, SDValue Chain, 2100 bool IsTailCall, bool Is64Bit, 2101 int FPDiff, DebugLoc dl) const { 2102 // Adjust the Return address stack slot. 2103 EVT VT = getPointerTy(); 2104 OutRetAddr = getReturnAddressFrameIndex(DAG); 2105 2106 // Load the "old" Return address. 2107 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(), 2108 false, false, false, 0); 2109 return SDValue(OutRetAddr.getNode(), 1); 2110} 2111 2112/// EmitTailCallStoreRetAddr - Emit a store of the return address if tail call 2113/// optimization is performed and it is required (FPDiff!=0). 2114static SDValue 2115EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 2116 SDValue Chain, SDValue RetAddrFrIdx, 2117 bool Is64Bit, int FPDiff, DebugLoc dl) { 2118 // Store the return address to the appropriate stack slot. 2119 if (!FPDiff) return Chain; 2120 // Calculate the new stack slot for the return address. 2121 int SlotSize = Is64Bit ? 8 : 4; 2122 int NewReturnAddrFI = 2123 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize, false); 2124 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 2125 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 2126 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, 2127 MachinePointerInfo::getFixedStack(NewReturnAddrFI), 2128 false, false, 0); 2129 return Chain; 2130} 2131 2132SDValue 2133X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee, 2134 CallingConv::ID CallConv, bool isVarArg, 2135 bool doesNotRet, bool &isTailCall, 2136 const SmallVectorImpl<ISD::OutputArg> &Outs, 2137 const SmallVectorImpl<SDValue> &OutVals, 2138 const SmallVectorImpl<ISD::InputArg> &Ins, 2139 DebugLoc dl, SelectionDAG &DAG, 2140 SmallVectorImpl<SDValue> &InVals) const { 2141 MachineFunction &MF = DAG.getMachineFunction(); 2142 bool Is64Bit = Subtarget->is64Bit(); 2143 bool IsWin64 = Subtarget->isTargetWin64(); 2144 bool IsWindows = Subtarget->isTargetWindows(); 2145 bool IsStructRet = CallIsStructReturn(Outs); 2146 bool IsSibcall = false; 2147 2148 if (MF.getTarget().Options.DisableTailCalls) 2149 isTailCall = false; 2150 2151 if (isTailCall) { 2152 // Check if it's really possible to do a tail call. 2153 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, 2154 isVarArg, IsStructRet, MF.getFunction()->hasStructRetAttr(), 2155 Outs, OutVals, Ins, DAG); 2156 2157 // Sibcalls are automatically detected tailcalls which do not require 2158 // ABI changes. 2159 if (!MF.getTarget().Options.GuaranteedTailCallOpt && isTailCall) 2160 IsSibcall = true; 2161 2162 if (isTailCall) 2163 ++NumTailCalls; 2164 } 2165 2166 assert(!(isVarArg && IsTailCallConvention(CallConv)) && 2167 "Var args not supported with calling convention fastcc or ghc"); 2168 2169 // Analyze operands of the call, assigning locations to each operand. 2170 SmallVector<CCValAssign, 16> ArgLocs; 2171 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 2172 ArgLocs, *DAG.getContext()); 2173 2174 // Allocate shadow area for Win64 2175 if (IsWin64) { 2176 CCInfo.AllocateStack(32, 8); 2177 } 2178 2179 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 2180 2181 // Get a count of how many bytes are to be pushed on the stack. 2182 unsigned NumBytes = CCInfo.getNextStackOffset(); 2183 if (IsSibcall) 2184 // This is a sibcall. The memory operands are available in caller's 2185 // own caller's stack. 2186 NumBytes = 0; 2187 else if (getTargetMachine().Options.GuaranteedTailCallOpt && 2188 IsTailCallConvention(CallConv)) 2189 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 2190 2191 int FPDiff = 0; 2192 if (isTailCall && !IsSibcall) { 2193 // Lower arguments at fp - stackoffset + fpdiff. 2194 unsigned NumBytesCallerPushed = 2195 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 2196 FPDiff = NumBytesCallerPushed - NumBytes; 2197 2198 // Set the delta of movement of the returnaddr stackslot. 2199 // But only set if delta is greater than previous delta. 2200 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 2201 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 2202 } 2203 2204 if (!IsSibcall) 2205 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true)); 2206 2207 SDValue RetAddrFrIdx; 2208 // Load return address for tail calls. 2209 if (isTailCall && FPDiff) 2210 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall, 2211 Is64Bit, FPDiff, dl); 2212 2213 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 2214 SmallVector<SDValue, 8> MemOpChains; 2215 SDValue StackPtr; 2216 2217 // Walk the register/memloc assignments, inserting copies/loads. In the case 2218 // of tail call optimization arguments are handle later. 2219 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2220 CCValAssign &VA = ArgLocs[i]; 2221 EVT RegVT = VA.getLocVT(); 2222 SDValue Arg = OutVals[i]; 2223 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2224 bool isByVal = Flags.isByVal(); 2225 2226 // Promote the value if needed. 2227 switch (VA.getLocInfo()) { 2228 default: llvm_unreachable("Unknown loc info!"); 2229 case CCValAssign::Full: break; 2230 case CCValAssign::SExt: 2231 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); 2232 break; 2233 case CCValAssign::ZExt: 2234 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); 2235 break; 2236 case CCValAssign::AExt: 2237 if (RegVT.isVector() && RegVT.getSizeInBits() == 128) { 2238 // Special case: passing MMX values in XMM registers. 2239 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 2240 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); 2241 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); 2242 } else 2243 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); 2244 break; 2245 case CCValAssign::BCvt: 2246 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg); 2247 break; 2248 case CCValAssign::Indirect: { 2249 // Store the argument. 2250 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 2251 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 2252 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot, 2253 MachinePointerInfo::getFixedStack(FI), 2254 false, false, 0); 2255 Arg = SpillSlot; 2256 break; 2257 } 2258 } 2259 2260 if (VA.isRegLoc()) { 2261 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 2262 if (isVarArg && IsWin64) { 2263 // Win64 ABI requires argument XMM reg to be copied to the corresponding 2264 // shadow reg if callee is a varargs function. 2265 unsigned ShadowReg = 0; 2266 switch (VA.getLocReg()) { 2267 case X86::XMM0: ShadowReg = X86::RCX; break; 2268 case X86::XMM1: ShadowReg = X86::RDX; break; 2269 case X86::XMM2: ShadowReg = X86::R8; break; 2270 case X86::XMM3: ShadowReg = X86::R9; break; 2271 } 2272 if (ShadowReg) 2273 RegsToPass.push_back(std::make_pair(ShadowReg, Arg)); 2274 } 2275 } else if (!IsSibcall && (!isTailCall || isByVal)) { 2276 assert(VA.isMemLoc()); 2277 if (StackPtr.getNode() == 0) 2278 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy()); 2279 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, 2280 dl, DAG, VA, Flags)); 2281 } 2282 } 2283 2284 if (!MemOpChains.empty()) 2285 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 2286 &MemOpChains[0], MemOpChains.size()); 2287 2288 // Build a sequence of copy-to-reg nodes chained together with token chain 2289 // and flag operands which copy the outgoing args into registers. 2290 SDValue InFlag; 2291 // Tail call byval lowering might overwrite argument registers so in case of 2292 // tail call optimization the copies to registers are lowered later. 2293 if (!isTailCall) 2294 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 2295 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 2296 RegsToPass[i].second, InFlag); 2297 InFlag = Chain.getValue(1); 2298 } 2299 2300 if (Subtarget->isPICStyleGOT()) { 2301 // ELF / PIC requires GOT in the EBX register before function calls via PLT 2302 // GOT pointer. 2303 if (!isTailCall) { 2304 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX, 2305 DAG.getNode(X86ISD::GlobalBaseReg, 2306 DebugLoc(), getPointerTy()), 2307 InFlag); 2308 InFlag = Chain.getValue(1); 2309 } else { 2310 // If we are tail calling and generating PIC/GOT style code load the 2311 // address of the callee into ECX. The value in ecx is used as target of 2312 // the tail jump. This is done to circumvent the ebx/callee-saved problem 2313 // for tail calls on PIC/GOT architectures. Normally we would just put the 2314 // address of GOT into ebx and then call target@PLT. But for tail calls 2315 // ebx would be restored (since ebx is callee saved) before jumping to the 2316 // target@PLT. 2317 2318 // Note: The actual moving to ECX is done further down. 2319 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 2320 if (G && !G->getGlobal()->hasHiddenVisibility() && 2321 !G->getGlobal()->hasProtectedVisibility()) 2322 Callee = LowerGlobalAddress(Callee, DAG); 2323 else if (isa<ExternalSymbolSDNode>(Callee)) 2324 Callee = LowerExternalSymbol(Callee, DAG); 2325 } 2326 } 2327 2328 if (Is64Bit && isVarArg && !IsWin64) { 2329 // From AMD64 ABI document: 2330 // For calls that may call functions that use varargs or stdargs 2331 // (prototype-less calls or calls to functions containing ellipsis (...) in 2332 // the declaration) %al is used as hidden argument to specify the number 2333 // of SSE registers used. The contents of %al do not need to match exactly 2334 // the number of registers, but must be an ubound on the number of SSE 2335 // registers used and is in the range 0 - 8 inclusive. 2336 2337 // Count the number of XMM registers allocated. 2338 static const uint16_t XMMArgRegs[] = { 2339 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 2340 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 2341 }; 2342 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 2343 assert((Subtarget->hasSSE1() || !NumXMMRegs) 2344 && "SSE registers cannot be used when SSE is disabled"); 2345 2346 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, 2347 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 2348 InFlag = Chain.getValue(1); 2349 } 2350 2351 2352 // For tail calls lower the arguments to the 'real' stack slot. 2353 if (isTailCall) { 2354 // Force all the incoming stack arguments to be loaded from the stack 2355 // before any new outgoing arguments are stored to the stack, because the 2356 // outgoing stack slots may alias the incoming argument stack slots, and 2357 // the alias isn't otherwise explicit. This is slightly more conservative 2358 // than necessary, because it means that each store effectively depends 2359 // on every argument instead of just those arguments it would clobber. 2360 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain); 2361 2362 SmallVector<SDValue, 8> MemOpChains2; 2363 SDValue FIN; 2364 int FI = 0; 2365 // Do not flag preceding copytoreg stuff together with the following stuff. 2366 InFlag = SDValue(); 2367 if (getTargetMachine().Options.GuaranteedTailCallOpt) { 2368 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2369 CCValAssign &VA = ArgLocs[i]; 2370 if (VA.isRegLoc()) 2371 continue; 2372 assert(VA.isMemLoc()); 2373 SDValue Arg = OutVals[i]; 2374 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2375 // Create frame index. 2376 int32_t Offset = VA.getLocMemOffset()+FPDiff; 2377 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 2378 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); 2379 FIN = DAG.getFrameIndex(FI, getPointerTy()); 2380 2381 if (Flags.isByVal()) { 2382 // Copy relative to framepointer. 2383 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 2384 if (StackPtr.getNode() == 0) 2385 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, 2386 getPointerTy()); 2387 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source); 2388 2389 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, 2390 ArgChain, 2391 Flags, DAG, dl)); 2392 } else { 2393 // Store relative to framepointer. 2394 MemOpChains2.push_back( 2395 DAG.getStore(ArgChain, dl, Arg, FIN, 2396 MachinePointerInfo::getFixedStack(FI), 2397 false, false, 0)); 2398 } 2399 } 2400 } 2401 2402 if (!MemOpChains2.empty()) 2403 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 2404 &MemOpChains2[0], MemOpChains2.size()); 2405 2406 // Copy arguments to their registers. 2407 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 2408 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 2409 RegsToPass[i].second, InFlag); 2410 InFlag = Chain.getValue(1); 2411 } 2412 InFlag =SDValue(); 2413 2414 // Store the return address to the appropriate stack slot. 2415 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 2416 FPDiff, dl); 2417 } 2418 2419 if (getTargetMachine().getCodeModel() == CodeModel::Large) { 2420 assert(Is64Bit && "Large code model is only legal in 64-bit mode."); 2421 // In the 64-bit large code model, we have to make all calls 2422 // through a register, since the call instruction's 32-bit 2423 // pc-relative offset may not be large enough to hold the whole 2424 // address. 2425 } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 2426 // If the callee is a GlobalAddress node (quite common, every direct call 2427 // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack 2428 // it. 2429 2430 // We should use extra load for direct calls to dllimported functions in 2431 // non-JIT mode. 2432 const GlobalValue *GV = G->getGlobal(); 2433 if (!GV->hasDLLImportLinkage()) { 2434 unsigned char OpFlags = 0; 2435 bool ExtraLoad = false; 2436 unsigned WrapperKind = ISD::DELETED_NODE; 2437 2438 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to 2439 // external symbols most go through the PLT in PIC mode. If the symbol 2440 // has hidden or protected visibility, or if it is static or local, then 2441 // we don't need to use the PLT - we can directly call it. 2442 if (Subtarget->isTargetELF() && 2443 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 2444 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) { 2445 OpFlags = X86II::MO_PLT; 2446 } else if (Subtarget->isPICStyleStubAny() && 2447 (GV->isDeclaration() || GV->isWeakForLinker()) && 2448 (!Subtarget->getTargetTriple().isMacOSX() || 2449 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) { 2450 // PC-relative references to external symbols should go through $stub, 2451 // unless we're building with the leopard linker or later, which 2452 // automatically synthesizes these stubs. 2453 OpFlags = X86II::MO_DARWIN_STUB; 2454 } else if (Subtarget->isPICStyleRIPRel() && 2455 isa<Function>(GV) && 2456 cast<Function>(GV)->hasFnAttr(Attribute::NonLazyBind)) { 2457 // If the function is marked as non-lazy, generate an indirect call 2458 // which loads from the GOT directly. This avoids runtime overhead 2459 // at the cost of eager binding (and one extra byte of encoding). 2460 OpFlags = X86II::MO_GOTPCREL; 2461 WrapperKind = X86ISD::WrapperRIP; 2462 ExtraLoad = true; 2463 } 2464 2465 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 2466 G->getOffset(), OpFlags); 2467 2468 // Add a wrapper if needed. 2469 if (WrapperKind != ISD::DELETED_NODE) 2470 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee); 2471 // Add extra indirection if needed. 2472 if (ExtraLoad) 2473 Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee, 2474 MachinePointerInfo::getGOT(), 2475 false, false, false, 0); 2476 } 2477 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 2478 unsigned char OpFlags = 0; 2479 2480 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to 2481 // external symbols should go through the PLT. 2482 if (Subtarget->isTargetELF() && 2483 getTargetMachine().getRelocationModel() == Reloc::PIC_) { 2484 OpFlags = X86II::MO_PLT; 2485 } else if (Subtarget->isPICStyleStubAny() && 2486 (!Subtarget->getTargetTriple().isMacOSX() || 2487 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) { 2488 // PC-relative references to external symbols should go through $stub, 2489 // unless we're building with the leopard linker or later, which 2490 // automatically synthesizes these stubs. 2491 OpFlags = X86II::MO_DARWIN_STUB; 2492 } 2493 2494 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(), 2495 OpFlags); 2496 } 2497 2498 // Returns a chain & a flag for retval copy to use. 2499 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2500 SmallVector<SDValue, 8> Ops; 2501 2502 if (!IsSibcall && isTailCall) { 2503 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), 2504 DAG.getIntPtrConstant(0, true), InFlag); 2505 InFlag = Chain.getValue(1); 2506 } 2507 2508 Ops.push_back(Chain); 2509 Ops.push_back(Callee); 2510 2511 if (isTailCall) 2512 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 2513 2514 // Add argument registers to the end of the list so that they are known live 2515 // into the call. 2516 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 2517 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 2518 RegsToPass[i].second.getValueType())); 2519 2520 // Add an implicit use GOT pointer in EBX. 2521 if (!isTailCall && Subtarget->isPICStyleGOT()) 2522 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 2523 2524 // Add an implicit use of AL for non-Windows x86 64-bit vararg functions. 2525 if (Is64Bit && isVarArg && !IsWin64) 2526 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 2527 2528 // Add a register mask operand representing the call-preserved registers. 2529 const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo(); 2530 const uint32_t *Mask = TRI->getCallPreservedMask(CallConv); 2531 assert(Mask && "Missing call preserved mask for calling convention"); 2532 Ops.push_back(DAG.getRegisterMask(Mask)); 2533 2534 if (InFlag.getNode()) 2535 Ops.push_back(InFlag); 2536 2537 if (isTailCall) { 2538 // We used to do: 2539 //// If this is the first return lowered for this function, add the regs 2540 //// to the liveout set for the function. 2541 // This isn't right, although it's probably harmless on x86; liveouts 2542 // should be computed from returns not tail calls. Consider a void 2543 // function making a tail call to a function returning int. 2544 return DAG.getNode(X86ISD::TC_RETURN, dl, 2545 NodeTys, &Ops[0], Ops.size()); 2546 } 2547 2548 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size()); 2549 InFlag = Chain.getValue(1); 2550 2551 // Create the CALLSEQ_END node. 2552 unsigned NumBytesForCalleeToPush; 2553 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, 2554 getTargetMachine().Options.GuaranteedTailCallOpt)) 2555 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 2556 else if (!Is64Bit && !IsTailCallConvention(CallConv) && !IsWindows && 2557 IsStructRet) 2558 // If this is a call to a struct-return function, the callee 2559 // pops the hidden struct pointer, so we have to push it back. 2560 // This is common for Darwin/X86, Linux & Mingw32 targets. 2561 // For MSVC Win32 targets, the caller pops the hidden struct pointer. 2562 NumBytesForCalleeToPush = 4; 2563 else 2564 NumBytesForCalleeToPush = 0; // Callee pops nothing. 2565 2566 // Returns a flag for retval copy to use. 2567 if (!IsSibcall) { 2568 Chain = DAG.getCALLSEQ_END(Chain, 2569 DAG.getIntPtrConstant(NumBytes, true), 2570 DAG.getIntPtrConstant(NumBytesForCalleeToPush, 2571 true), 2572 InFlag); 2573 InFlag = Chain.getValue(1); 2574 } 2575 2576 // Handle result values, copying them out of physregs into vregs that we 2577 // return. 2578 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 2579 Ins, dl, DAG, InVals); 2580} 2581 2582 2583//===----------------------------------------------------------------------===// 2584// Fast Calling Convention (tail call) implementation 2585//===----------------------------------------------------------------------===// 2586 2587// Like std call, callee cleans arguments, convention except that ECX is 2588// reserved for storing the tail called function address. Only 2 registers are 2589// free for argument passing (inreg). Tail call optimization is performed 2590// provided: 2591// * tailcallopt is enabled 2592// * caller/callee are fastcc 2593// On X86_64 architecture with GOT-style position independent code only local 2594// (within module) calls are supported at the moment. 2595// To keep the stack aligned according to platform abi the function 2596// GetAlignedArgumentStackSize ensures that argument delta is always multiples 2597// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 2598// If a tail called function callee has more arguments than the caller the 2599// caller needs to make sure that there is room to move the RETADDR to. This is 2600// achieved by reserving an area the size of the argument delta right after the 2601// original REtADDR, but before the saved framepointer or the spilled registers 2602// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 2603// stack layout: 2604// arg1 2605// arg2 2606// RETADDR 2607// [ new RETADDR 2608// move area ] 2609// (possible EBP) 2610// ESI 2611// EDI 2612// local1 .. 2613 2614/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 2615/// for a 16 byte align requirement. 2616unsigned 2617X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 2618 SelectionDAG& DAG) const { 2619 MachineFunction &MF = DAG.getMachineFunction(); 2620 const TargetMachine &TM = MF.getTarget(); 2621 const TargetFrameLowering &TFI = *TM.getFrameLowering(); 2622 unsigned StackAlignment = TFI.getStackAlignment(); 2623 uint64_t AlignMask = StackAlignment - 1; 2624 int64_t Offset = StackSize; 2625 uint64_t SlotSize = TD->getPointerSize(); 2626 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 2627 // Number smaller than 12 so just add the difference. 2628 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 2629 } else { 2630 // Mask out lower bits, add stackalignment once plus the 12 bytes. 2631 Offset = ((~AlignMask) & Offset) + StackAlignment + 2632 (StackAlignment-SlotSize); 2633 } 2634 return Offset; 2635} 2636 2637/// MatchingStackOffset - Return true if the given stack call argument is 2638/// already available in the same position (relatively) of the caller's 2639/// incoming argument stack. 2640static 2641bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, 2642 MachineFrameInfo *MFI, const MachineRegisterInfo *MRI, 2643 const X86InstrInfo *TII) { 2644 unsigned Bytes = Arg.getValueType().getSizeInBits() / 8; 2645 int FI = INT_MAX; 2646 if (Arg.getOpcode() == ISD::CopyFromReg) { 2647 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); 2648 if (!TargetRegisterInfo::isVirtualRegister(VR)) 2649 return false; 2650 MachineInstr *Def = MRI->getVRegDef(VR); 2651 if (!Def) 2652 return false; 2653 if (!Flags.isByVal()) { 2654 if (!TII->isLoadFromStackSlot(Def, FI)) 2655 return false; 2656 } else { 2657 unsigned Opcode = Def->getOpcode(); 2658 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r) && 2659 Def->getOperand(1).isFI()) { 2660 FI = Def->getOperand(1).getIndex(); 2661 Bytes = Flags.getByValSize(); 2662 } else 2663 return false; 2664 } 2665 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) { 2666 if (Flags.isByVal()) 2667 // ByVal argument is passed in as a pointer but it's now being 2668 // dereferenced. e.g. 2669 // define @foo(%struct.X* %A) { 2670 // tail call @bar(%struct.X* byval %A) 2671 // } 2672 return false; 2673 SDValue Ptr = Ld->getBasePtr(); 2674 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr); 2675 if (!FINode) 2676 return false; 2677 FI = FINode->getIndex(); 2678 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) { 2679 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg); 2680 FI = FINode->getIndex(); 2681 Bytes = Flags.getByValSize(); 2682 } else 2683 return false; 2684 2685 assert(FI != INT_MAX); 2686 if (!MFI->isFixedObjectIndex(FI)) 2687 return false; 2688 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI); 2689} 2690 2691/// IsEligibleForTailCallOptimization - Check whether the call is eligible 2692/// for tail call optimization. Targets which want to do tail call 2693/// optimization should implement this function. 2694bool 2695X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 2696 CallingConv::ID CalleeCC, 2697 bool isVarArg, 2698 bool isCalleeStructRet, 2699 bool isCallerStructRet, 2700 const SmallVectorImpl<ISD::OutputArg> &Outs, 2701 const SmallVectorImpl<SDValue> &OutVals, 2702 const SmallVectorImpl<ISD::InputArg> &Ins, 2703 SelectionDAG& DAG) const { 2704 if (!IsTailCallConvention(CalleeCC) && 2705 CalleeCC != CallingConv::C) 2706 return false; 2707 2708 // If -tailcallopt is specified, make fastcc functions tail-callable. 2709 const MachineFunction &MF = DAG.getMachineFunction(); 2710 const Function *CallerF = DAG.getMachineFunction().getFunction(); 2711 CallingConv::ID CallerCC = CallerF->getCallingConv(); 2712 bool CCMatch = CallerCC == CalleeCC; 2713 2714 if (getTargetMachine().Options.GuaranteedTailCallOpt) { 2715 if (IsTailCallConvention(CalleeCC) && CCMatch) 2716 return true; 2717 return false; 2718 } 2719 2720 // Look for obvious safe cases to perform tail call optimization that do not 2721 // require ABI changes. This is what gcc calls sibcall. 2722 2723 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to 2724 // emit a special epilogue. 2725 if (RegInfo->needsStackRealignment(MF)) 2726 return false; 2727 2728 // Also avoid sibcall optimization if either caller or callee uses struct 2729 // return semantics. 2730 if (isCalleeStructRet || isCallerStructRet) 2731 return false; 2732 2733 // An stdcall caller is expected to clean up its arguments; the callee 2734 // isn't going to do that. 2735 if (!CCMatch && CallerCC==CallingConv::X86_StdCall) 2736 return false; 2737 2738 // Do not sibcall optimize vararg calls unless all arguments are passed via 2739 // registers. 2740 if (isVarArg && !Outs.empty()) { 2741 2742 // Optimizing for varargs on Win64 is unlikely to be safe without 2743 // additional testing. 2744 if (Subtarget->isTargetWin64()) 2745 return false; 2746 2747 SmallVector<CCValAssign, 16> ArgLocs; 2748 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), 2749 getTargetMachine(), ArgLocs, *DAG.getContext()); 2750 2751 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 2752 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) 2753 if (!ArgLocs[i].isRegLoc()) 2754 return false; 2755 } 2756 2757 // If the call result is in ST0 / ST1, it needs to be popped off the x87 2758 // stack. Therefore, if it's not used by the call it is not safe to optimize 2759 // this into a sibcall. 2760 bool Unused = false; 2761 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 2762 if (!Ins[i].Used) { 2763 Unused = true; 2764 break; 2765 } 2766 } 2767 if (Unused) { 2768 SmallVector<CCValAssign, 16> RVLocs; 2769 CCState CCInfo(CalleeCC, false, DAG.getMachineFunction(), 2770 getTargetMachine(), RVLocs, *DAG.getContext()); 2771 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 2772 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 2773 CCValAssign &VA = RVLocs[i]; 2774 if (VA.getLocReg() == X86::ST0 || VA.getLocReg() == X86::ST1) 2775 return false; 2776 } 2777 } 2778 2779 // If the calling conventions do not match, then we'd better make sure the 2780 // results are returned in the same way as what the caller expects. 2781 if (!CCMatch) { 2782 SmallVector<CCValAssign, 16> RVLocs1; 2783 CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), 2784 getTargetMachine(), RVLocs1, *DAG.getContext()); 2785 CCInfo1.AnalyzeCallResult(Ins, RetCC_X86); 2786 2787 SmallVector<CCValAssign, 16> RVLocs2; 2788 CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), 2789 getTargetMachine(), RVLocs2, *DAG.getContext()); 2790 CCInfo2.AnalyzeCallResult(Ins, RetCC_X86); 2791 2792 if (RVLocs1.size() != RVLocs2.size()) 2793 return false; 2794 for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) { 2795 if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc()) 2796 return false; 2797 if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo()) 2798 return false; 2799 if (RVLocs1[i].isRegLoc()) { 2800 if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg()) 2801 return false; 2802 } else { 2803 if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset()) 2804 return false; 2805 } 2806 } 2807 } 2808 2809 // If the callee takes no arguments then go on to check the results of the 2810 // call. 2811 if (!Outs.empty()) { 2812 // Check if stack adjustment is needed. For now, do not do this if any 2813 // argument is passed on the stack. 2814 SmallVector<CCValAssign, 16> ArgLocs; 2815 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), 2816 getTargetMachine(), ArgLocs, *DAG.getContext()); 2817 2818 // Allocate shadow area for Win64 2819 if (Subtarget->isTargetWin64()) { 2820 CCInfo.AllocateStack(32, 8); 2821 } 2822 2823 CCInfo.AnalyzeCallOperands(Outs, CC_X86); 2824 if (CCInfo.getNextStackOffset()) { 2825 MachineFunction &MF = DAG.getMachineFunction(); 2826 if (MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn()) 2827 return false; 2828 2829 // Check if the arguments are already laid out in the right way as 2830 // the caller's fixed stack objects. 2831 MachineFrameInfo *MFI = MF.getFrameInfo(); 2832 const MachineRegisterInfo *MRI = &MF.getRegInfo(); 2833 const X86InstrInfo *TII = 2834 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 2835 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2836 CCValAssign &VA = ArgLocs[i]; 2837 SDValue Arg = OutVals[i]; 2838 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2839 if (VA.getLocInfo() == CCValAssign::Indirect) 2840 return false; 2841 if (!VA.isRegLoc()) { 2842 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, 2843 MFI, MRI, TII)) 2844 return false; 2845 } 2846 } 2847 } 2848 2849 // If the tailcall address may be in a register, then make sure it's 2850 // possible to register allocate for it. In 32-bit, the call address can 2851 // only target EAX, EDX, or ECX since the tail call must be scheduled after 2852 // callee-saved registers are restored. These happen to be the same 2853 // registers used to pass 'inreg' arguments so watch out for those. 2854 if (!Subtarget->is64Bit() && 2855 !isa<GlobalAddressSDNode>(Callee) && 2856 !isa<ExternalSymbolSDNode>(Callee)) { 2857 unsigned NumInRegs = 0; 2858 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2859 CCValAssign &VA = ArgLocs[i]; 2860 if (!VA.isRegLoc()) 2861 continue; 2862 unsigned Reg = VA.getLocReg(); 2863 switch (Reg) { 2864 default: break; 2865 case X86::EAX: case X86::EDX: case X86::ECX: 2866 if (++NumInRegs == 3) 2867 return false; 2868 break; 2869 } 2870 } 2871 } 2872 } 2873 2874 return true; 2875} 2876 2877FastISel * 2878X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo) const { 2879 return X86::createFastISel(funcInfo); 2880} 2881 2882 2883//===----------------------------------------------------------------------===// 2884// Other Lowering Hooks 2885//===----------------------------------------------------------------------===// 2886 2887static bool MayFoldLoad(SDValue Op) { 2888 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode()); 2889} 2890 2891static bool MayFoldIntoStore(SDValue Op) { 2892 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); 2893} 2894 2895static bool isTargetShuffle(unsigned Opcode) { 2896 switch(Opcode) { 2897 default: return false; 2898 case X86ISD::PSHUFD: 2899 case X86ISD::PSHUFHW: 2900 case X86ISD::PSHUFLW: 2901 case X86ISD::SHUFP: 2902 case X86ISD::PALIGN: 2903 case X86ISD::MOVLHPS: 2904 case X86ISD::MOVLHPD: 2905 case X86ISD::MOVHLPS: 2906 case X86ISD::MOVLPS: 2907 case X86ISD::MOVLPD: 2908 case X86ISD::MOVSHDUP: 2909 case X86ISD::MOVSLDUP: 2910 case X86ISD::MOVDDUP: 2911 case X86ISD::MOVSS: 2912 case X86ISD::MOVSD: 2913 case X86ISD::UNPCKL: 2914 case X86ISD::UNPCKH: 2915 case X86ISD::VPERMILP: 2916 case X86ISD::VPERM2X128: 2917 case X86ISD::VPERMI: 2918 return true; 2919 } 2920} 2921 2922static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2923 SDValue V1, SelectionDAG &DAG) { 2924 switch(Opc) { 2925 default: llvm_unreachable("Unknown x86 shuffle node"); 2926 case X86ISD::MOVSHDUP: 2927 case X86ISD::MOVSLDUP: 2928 case X86ISD::MOVDDUP: 2929 return DAG.getNode(Opc, dl, VT, V1); 2930 } 2931} 2932 2933static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2934 SDValue V1, unsigned TargetMask, 2935 SelectionDAG &DAG) { 2936 switch(Opc) { 2937 default: llvm_unreachable("Unknown x86 shuffle node"); 2938 case X86ISD::PSHUFD: 2939 case X86ISD::PSHUFHW: 2940 case X86ISD::PSHUFLW: 2941 case X86ISD::VPERMILP: 2942 case X86ISD::VPERMI: 2943 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8)); 2944 } 2945} 2946 2947static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2948 SDValue V1, SDValue V2, unsigned TargetMask, 2949 SelectionDAG &DAG) { 2950 switch(Opc) { 2951 default: llvm_unreachable("Unknown x86 shuffle node"); 2952 case X86ISD::PALIGN: 2953 case X86ISD::SHUFP: 2954 case X86ISD::VPERM2X128: 2955 return DAG.getNode(Opc, dl, VT, V1, V2, 2956 DAG.getConstant(TargetMask, MVT::i8)); 2957 } 2958} 2959 2960static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT, 2961 SDValue V1, SDValue V2, SelectionDAG &DAG) { 2962 switch(Opc) { 2963 default: llvm_unreachable("Unknown x86 shuffle node"); 2964 case X86ISD::MOVLHPS: 2965 case X86ISD::MOVLHPD: 2966 case X86ISD::MOVHLPS: 2967 case X86ISD::MOVLPS: 2968 case X86ISD::MOVLPD: 2969 case X86ISD::MOVSS: 2970 case X86ISD::MOVSD: 2971 case X86ISD::UNPCKL: 2972 case X86ISD::UNPCKH: 2973 return DAG.getNode(Opc, dl, VT, V1, V2); 2974 } 2975} 2976 2977SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const { 2978 MachineFunction &MF = DAG.getMachineFunction(); 2979 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 2980 int ReturnAddrIndex = FuncInfo->getRAIndex(); 2981 2982 if (ReturnAddrIndex == 0) { 2983 // Set up a frame object for the return address. 2984 uint64_t SlotSize = TD->getPointerSize(); 2985 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize, 2986 false); 2987 FuncInfo->setRAIndex(ReturnAddrIndex); 2988 } 2989 2990 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 2991} 2992 2993 2994bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, 2995 bool hasSymbolicDisplacement) { 2996 // Offset should fit into 32 bit immediate field. 2997 if (!isInt<32>(Offset)) 2998 return false; 2999 3000 // If we don't have a symbolic displacement - we don't have any extra 3001 // restrictions. 3002 if (!hasSymbolicDisplacement) 3003 return true; 3004 3005 // FIXME: Some tweaks might be needed for medium code model. 3006 if (M != CodeModel::Small && M != CodeModel::Kernel) 3007 return false; 3008 3009 // For small code model we assume that latest object is 16MB before end of 31 3010 // bits boundary. We may also accept pretty large negative constants knowing 3011 // that all objects are in the positive half of address space. 3012 if (M == CodeModel::Small && Offset < 16*1024*1024) 3013 return true; 3014 3015 // For kernel code model we know that all object resist in the negative half 3016 // of 32bits address space. We may not accept negative offsets, since they may 3017 // be just off and we may accept pretty large positive ones. 3018 if (M == CodeModel::Kernel && Offset > 0) 3019 return true; 3020 3021 return false; 3022} 3023 3024/// isCalleePop - Determines whether the callee is required to pop its 3025/// own arguments. Callee pop is necessary to support tail calls. 3026bool X86::isCalleePop(CallingConv::ID CallingConv, 3027 bool is64Bit, bool IsVarArg, bool TailCallOpt) { 3028 if (IsVarArg) 3029 return false; 3030 3031 switch (CallingConv) { 3032 default: 3033 return false; 3034 case CallingConv::X86_StdCall: 3035 return !is64Bit; 3036 case CallingConv::X86_FastCall: 3037 return !is64Bit; 3038 case CallingConv::X86_ThisCall: 3039 return !is64Bit; 3040 case CallingConv::Fast: 3041 return TailCallOpt; 3042 case CallingConv::GHC: 3043 return TailCallOpt; 3044 } 3045} 3046 3047/// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 3048/// specific condition code, returning the condition code and the LHS/RHS of the 3049/// comparison to make. 3050static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 3051 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) { 3052 if (!isFP) { 3053 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 3054 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 3055 // X > -1 -> X == 0, jump !sign. 3056 RHS = DAG.getConstant(0, RHS.getValueType()); 3057 return X86::COND_NS; 3058 } 3059 if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 3060 // X < 0 -> X == 0, jump on sign. 3061 return X86::COND_S; 3062 } 3063 if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) { 3064 // X < 1 -> X <= 0 3065 RHS = DAG.getConstant(0, RHS.getValueType()); 3066 return X86::COND_LE; 3067 } 3068 } 3069 3070 switch (SetCCOpcode) { 3071 default: llvm_unreachable("Invalid integer condition!"); 3072 case ISD::SETEQ: return X86::COND_E; 3073 case ISD::SETGT: return X86::COND_G; 3074 case ISD::SETGE: return X86::COND_GE; 3075 case ISD::SETLT: return X86::COND_L; 3076 case ISD::SETLE: return X86::COND_LE; 3077 case ISD::SETNE: return X86::COND_NE; 3078 case ISD::SETULT: return X86::COND_B; 3079 case ISD::SETUGT: return X86::COND_A; 3080 case ISD::SETULE: return X86::COND_BE; 3081 case ISD::SETUGE: return X86::COND_AE; 3082 } 3083 } 3084 3085 // First determine if it is required or is profitable to flip the operands. 3086 3087 // If LHS is a foldable load, but RHS is not, flip the condition. 3088 if (ISD::isNON_EXTLoad(LHS.getNode()) && 3089 !ISD::isNON_EXTLoad(RHS.getNode())) { 3090 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); 3091 std::swap(LHS, RHS); 3092 } 3093 3094 switch (SetCCOpcode) { 3095 default: break; 3096 case ISD::SETOLT: 3097 case ISD::SETOLE: 3098 case ISD::SETUGT: 3099 case ISD::SETUGE: 3100 std::swap(LHS, RHS); 3101 break; 3102 } 3103 3104 // On a floating point condition, the flags are set as follows: 3105 // ZF PF CF op 3106 // 0 | 0 | 0 | X > Y 3107 // 0 | 0 | 1 | X < Y 3108 // 1 | 0 | 0 | X == Y 3109 // 1 | 1 | 1 | unordered 3110 switch (SetCCOpcode) { 3111 default: llvm_unreachable("Condcode should be pre-legalized away"); 3112 case ISD::SETUEQ: 3113 case ISD::SETEQ: return X86::COND_E; 3114 case ISD::SETOLT: // flipped 3115 case ISD::SETOGT: 3116 case ISD::SETGT: return X86::COND_A; 3117 case ISD::SETOLE: // flipped 3118 case ISD::SETOGE: 3119 case ISD::SETGE: return X86::COND_AE; 3120 case ISD::SETUGT: // flipped 3121 case ISD::SETULT: 3122 case ISD::SETLT: return X86::COND_B; 3123 case ISD::SETUGE: // flipped 3124 case ISD::SETULE: 3125 case ISD::SETLE: return X86::COND_BE; 3126 case ISD::SETONE: 3127 case ISD::SETNE: return X86::COND_NE; 3128 case ISD::SETUO: return X86::COND_P; 3129 case ISD::SETO: return X86::COND_NP; 3130 case ISD::SETOEQ: 3131 case ISD::SETUNE: return X86::COND_INVALID; 3132 } 3133} 3134 3135/// hasFPCMov - is there a floating point cmov for the specific X86 condition 3136/// code. Current x86 isa includes the following FP cmov instructions: 3137/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 3138static bool hasFPCMov(unsigned X86CC) { 3139 switch (X86CC) { 3140 default: 3141 return false; 3142 case X86::COND_B: 3143 case X86::COND_BE: 3144 case X86::COND_E: 3145 case X86::COND_P: 3146 case X86::COND_A: 3147 case X86::COND_AE: 3148 case X86::COND_NE: 3149 case X86::COND_NP: 3150 return true; 3151 } 3152} 3153 3154/// isFPImmLegal - Returns true if the target can instruction select the 3155/// specified FP immediate natively. If false, the legalizer will 3156/// materialize the FP immediate as a load from a constant pool. 3157bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 3158 for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) { 3159 if (Imm.bitwiseIsEqual(LegalFPImmediates[i])) 3160 return true; 3161 } 3162 return false; 3163} 3164 3165/// isUndefOrInRange - Return true if Val is undef or if its value falls within 3166/// the specified range (L, H]. 3167static bool isUndefOrInRange(int Val, int Low, int Hi) { 3168 return (Val < 0) || (Val >= Low && Val < Hi); 3169} 3170 3171/// isUndefOrEqual - Val is either less than zero (undef) or equal to the 3172/// specified value. 3173static bool isUndefOrEqual(int Val, int CmpVal) { 3174 if (Val < 0 || Val == CmpVal) 3175 return true; 3176 return false; 3177} 3178 3179/// isSequentialOrUndefInRange - Return true if every element in Mask, begining 3180/// from position Pos and ending in Pos+Size, falls within the specified 3181/// sequential range (L, L+Pos]. or is undef. 3182static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, 3183 unsigned Pos, unsigned Size, int Low) { 3184 for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low) 3185 if (!isUndefOrEqual(Mask[i], Low)) 3186 return false; 3187 return true; 3188} 3189 3190/// isPSHUFDMask - Return true if the node specifies a shuffle of elements that 3191/// is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference 3192/// the second operand. 3193static bool isPSHUFDMask(ArrayRef<int> Mask, EVT VT) { 3194 if (VT == MVT::v4f32 || VT == MVT::v4i32 ) 3195 return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4); 3196 if (VT == MVT::v2f64 || VT == MVT::v2i64) 3197 return (Mask[0] < 2 && Mask[1] < 2); 3198 return false; 3199} 3200 3201/// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that 3202/// is suitable for input to PSHUFHW. 3203static bool isPSHUFHWMask(ArrayRef<int> Mask, EVT VT, bool HasAVX2) { 3204 if (VT != MVT::v8i16 && (!HasAVX2 || VT != MVT::v16i16)) 3205 return false; 3206 3207 // Lower quadword copied in order or undef. 3208 if (!isSequentialOrUndefInRange(Mask, 0, 4, 0)) 3209 return false; 3210 3211 // Upper quadword shuffled. 3212 for (unsigned i = 4; i != 8; ++i) 3213 if (!isUndefOrInRange(Mask[i], 4, 8)) 3214 return false; 3215 3216 if (VT == MVT::v16i16) { 3217 // Lower quadword copied in order or undef. 3218 if (!isSequentialOrUndefInRange(Mask, 8, 4, 8)) 3219 return false; 3220 3221 // Upper quadword shuffled. 3222 for (unsigned i = 12; i != 16; ++i) 3223 if (!isUndefOrInRange(Mask[i], 12, 16)) 3224 return false; 3225 } 3226 3227 return true; 3228} 3229 3230/// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that 3231/// is suitable for input to PSHUFLW. 3232static bool isPSHUFLWMask(ArrayRef<int> Mask, EVT VT, bool HasAVX2) { 3233 if (VT != MVT::v8i16 && (!HasAVX2 || VT != MVT::v16i16)) 3234 return false; 3235 3236 // Upper quadword copied in order. 3237 if (!isSequentialOrUndefInRange(Mask, 4, 4, 4)) 3238 return false; 3239 3240 // Lower quadword shuffled. 3241 for (unsigned i = 0; i != 4; ++i) 3242 if (!isUndefOrInRange(Mask[i], 0, 4)) 3243 return false; 3244 3245 if (VT == MVT::v16i16) { 3246 // Upper quadword copied in order. 3247 if (!isSequentialOrUndefInRange(Mask, 12, 4, 12)) 3248 return false; 3249 3250 // Lower quadword shuffled. 3251 for (unsigned i = 8; i != 12; ++i) 3252 if (!isUndefOrInRange(Mask[i], 8, 12)) 3253 return false; 3254 } 3255 3256 return true; 3257} 3258 3259/// isPALIGNRMask - Return true if the node specifies a shuffle of elements that 3260/// is suitable for input to PALIGNR. 3261static bool isPALIGNRMask(ArrayRef<int> Mask, EVT VT, 3262 const X86Subtarget *Subtarget) { 3263 if ((VT.getSizeInBits() == 128 && !Subtarget->hasSSSE3()) || 3264 (VT.getSizeInBits() == 256 && !Subtarget->hasAVX2())) 3265 return false; 3266 3267 unsigned NumElts = VT.getVectorNumElements(); 3268 unsigned NumLanes = VT.getSizeInBits()/128; 3269 unsigned NumLaneElts = NumElts/NumLanes; 3270 3271 // Do not handle 64-bit element shuffles with palignr. 3272 if (NumLaneElts == 2) 3273 return false; 3274 3275 for (unsigned l = 0; l != NumElts; l+=NumLaneElts) { 3276 unsigned i; 3277 for (i = 0; i != NumLaneElts; ++i) { 3278 if (Mask[i+l] >= 0) 3279 break; 3280 } 3281 3282 // Lane is all undef, go to next lane 3283 if (i == NumLaneElts) 3284 continue; 3285 3286 int Start = Mask[i+l]; 3287 3288 // Make sure its in this lane in one of the sources 3289 if (!isUndefOrInRange(Start, l, l+NumLaneElts) && 3290 !isUndefOrInRange(Start, l+NumElts, l+NumElts+NumLaneElts)) 3291 return false; 3292 3293 // If not lane 0, then we must match lane 0 3294 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Start, Mask[i]+l)) 3295 return false; 3296 3297 // Correct second source to be contiguous with first source 3298 if (Start >= (int)NumElts) 3299 Start -= NumElts - NumLaneElts; 3300 3301 // Make sure we're shifting in the right direction. 3302 if (Start <= (int)(i+l)) 3303 return false; 3304 3305 Start -= i; 3306 3307 // Check the rest of the elements to see if they are consecutive. 3308 for (++i; i != NumLaneElts; ++i) { 3309 int Idx = Mask[i+l]; 3310 3311 // Make sure its in this lane 3312 if (!isUndefOrInRange(Idx, l, l+NumLaneElts) && 3313 !isUndefOrInRange(Idx, l+NumElts, l+NumElts+NumLaneElts)) 3314 return false; 3315 3316 // If not lane 0, then we must match lane 0 3317 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Idx, Mask[i]+l)) 3318 return false; 3319 3320 if (Idx >= (int)NumElts) 3321 Idx -= NumElts - NumLaneElts; 3322 3323 if (!isUndefOrEqual(Idx, Start+i)) 3324 return false; 3325 3326 } 3327 } 3328 3329 return true; 3330} 3331 3332/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 3333/// the two vector operands have swapped position. 3334static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, 3335 unsigned NumElems) { 3336 for (unsigned i = 0; i != NumElems; ++i) { 3337 int idx = Mask[i]; 3338 if (idx < 0) 3339 continue; 3340 else if (idx < (int)NumElems) 3341 Mask[i] = idx + NumElems; 3342 else 3343 Mask[i] = idx - NumElems; 3344 } 3345} 3346 3347/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 3348/// specifies a shuffle of elements that is suitable for input to 128/256-bit 3349/// SHUFPS and SHUFPD. If Commuted is true, then it checks for sources to be 3350/// reverse of what x86 shuffles want. 3351static bool isSHUFPMask(ArrayRef<int> Mask, EVT VT, bool HasAVX, 3352 bool Commuted = false) { 3353 if (!HasAVX && VT.getSizeInBits() == 256) 3354 return false; 3355 3356 unsigned NumElems = VT.getVectorNumElements(); 3357 unsigned NumLanes = VT.getSizeInBits()/128; 3358 unsigned NumLaneElems = NumElems/NumLanes; 3359 3360 if (NumLaneElems != 2 && NumLaneElems != 4) 3361 return false; 3362 3363 // VSHUFPSY divides the resulting vector into 4 chunks. 3364 // The sources are also splitted into 4 chunks, and each destination 3365 // chunk must come from a different source chunk. 3366 // 3367 // SRC1 => X7 X6 X5 X4 X3 X2 X1 X0 3368 // SRC2 => Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y9 3369 // 3370 // DST => Y7..Y4, Y7..Y4, X7..X4, X7..X4, 3371 // Y3..Y0, Y3..Y0, X3..X0, X3..X0 3372 // 3373 // VSHUFPDY divides the resulting vector into 4 chunks. 3374 // The sources are also splitted into 4 chunks, and each destination 3375 // chunk must come from a different source chunk. 3376 // 3377 // SRC1 => X3 X2 X1 X0 3378 // SRC2 => Y3 Y2 Y1 Y0 3379 // 3380 // DST => Y3..Y2, X3..X2, Y1..Y0, X1..X0 3381 // 3382 unsigned HalfLaneElems = NumLaneElems/2; 3383 for (unsigned l = 0; l != NumElems; l += NumLaneElems) { 3384 for (unsigned i = 0; i != NumLaneElems; ++i) { 3385 int Idx = Mask[i+l]; 3386 unsigned RngStart = l + ((Commuted == (i<HalfLaneElems)) ? NumElems : 0); 3387 if (!isUndefOrInRange(Idx, RngStart, RngStart+NumLaneElems)) 3388 return false; 3389 // For VSHUFPSY, the mask of the second half must be the same as the 3390 // first but with the appropriate offsets. This works in the same way as 3391 // VPERMILPS works with masks. 3392 if (NumElems != 8 || l == 0 || Mask[i] < 0) 3393 continue; 3394 if (!isUndefOrEqual(Idx, Mask[i]+l)) 3395 return false; 3396 } 3397 } 3398 3399 return true; 3400} 3401 3402/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 3403/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 3404static bool isMOVHLPSMask(ArrayRef<int> Mask, EVT VT) { 3405 unsigned NumElems = VT.getVectorNumElements(); 3406 3407 if (VT.getSizeInBits() != 128) 3408 return false; 3409 3410 if (NumElems != 4) 3411 return false; 3412 3413 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 3414 return isUndefOrEqual(Mask[0], 6) && 3415 isUndefOrEqual(Mask[1], 7) && 3416 isUndefOrEqual(Mask[2], 2) && 3417 isUndefOrEqual(Mask[3], 3); 3418} 3419 3420/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 3421/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 3422/// <2, 3, 2, 3> 3423static bool isMOVHLPS_v_undef_Mask(ArrayRef<int> Mask, EVT VT) { 3424 unsigned NumElems = VT.getVectorNumElements(); 3425 3426 if (VT.getSizeInBits() != 128) 3427 return false; 3428 3429 if (NumElems != 4) 3430 return false; 3431 3432 return isUndefOrEqual(Mask[0], 2) && 3433 isUndefOrEqual(Mask[1], 3) && 3434 isUndefOrEqual(Mask[2], 2) && 3435 isUndefOrEqual(Mask[3], 3); 3436} 3437 3438/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 3439/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 3440static bool isMOVLPMask(ArrayRef<int> Mask, EVT VT) { 3441 if (VT.getSizeInBits() != 128) 3442 return false; 3443 3444 unsigned NumElems = VT.getVectorNumElements(); 3445 3446 if (NumElems != 2 && NumElems != 4) 3447 return false; 3448 3449 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 3450 if (!isUndefOrEqual(Mask[i], i + NumElems)) 3451 return false; 3452 3453 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i) 3454 if (!isUndefOrEqual(Mask[i], i)) 3455 return false; 3456 3457 return true; 3458} 3459 3460/// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand 3461/// specifies a shuffle of elements that is suitable for input to MOVLHPS. 3462static bool isMOVLHPSMask(ArrayRef<int> Mask, EVT VT) { 3463 unsigned NumElems = VT.getVectorNumElements(); 3464 3465 if ((NumElems != 2 && NumElems != 4) 3466 || VT.getSizeInBits() > 128) 3467 return false; 3468 3469 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 3470 if (!isUndefOrEqual(Mask[i], i)) 3471 return false; 3472 3473 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 3474 if (!isUndefOrEqual(Mask[i + e], i + NumElems)) 3475 return false; 3476 3477 return true; 3478} 3479 3480/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 3481/// specifies a shuffle of elements that is suitable for input to UNPCKL. 3482static bool isUNPCKLMask(ArrayRef<int> Mask, EVT VT, 3483 bool HasAVX2, bool V2IsSplat = false) { 3484 unsigned NumElts = VT.getVectorNumElements(); 3485 3486 assert((VT.is128BitVector() || VT.is256BitVector()) && 3487 "Unsupported vector type for unpckh"); 3488 3489 if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8 && 3490 (!HasAVX2 || (NumElts != 16 && NumElts != 32))) 3491 return false; 3492 3493 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate 3494 // independently on 128-bit lanes. 3495 unsigned NumLanes = VT.getSizeInBits()/128; 3496 unsigned NumLaneElts = NumElts/NumLanes; 3497 3498 for (unsigned l = 0; l != NumLanes; ++l) { 3499 for (unsigned i = l*NumLaneElts, j = l*NumLaneElts; 3500 i != (l+1)*NumLaneElts; 3501 i += 2, ++j) { 3502 int BitI = Mask[i]; 3503 int BitI1 = Mask[i+1]; 3504 if (!isUndefOrEqual(BitI, j)) 3505 return false; 3506 if (V2IsSplat) { 3507 if (!isUndefOrEqual(BitI1, NumElts)) 3508 return false; 3509 } else { 3510 if (!isUndefOrEqual(BitI1, j + NumElts)) 3511 return false; 3512 } 3513 } 3514 } 3515 3516 return true; 3517} 3518 3519/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 3520/// specifies a shuffle of elements that is suitable for input to UNPCKH. 3521static bool isUNPCKHMask(ArrayRef<int> Mask, EVT VT, 3522 bool HasAVX2, bool V2IsSplat = false) { 3523 unsigned NumElts = VT.getVectorNumElements(); 3524 3525 assert((VT.is128BitVector() || VT.is256BitVector()) && 3526 "Unsupported vector type for unpckh"); 3527 3528 if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8 && 3529 (!HasAVX2 || (NumElts != 16 && NumElts != 32))) 3530 return false; 3531 3532 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate 3533 // independently on 128-bit lanes. 3534 unsigned NumLanes = VT.getSizeInBits()/128; 3535 unsigned NumLaneElts = NumElts/NumLanes; 3536 3537 for (unsigned l = 0; l != NumLanes; ++l) { 3538 for (unsigned i = l*NumLaneElts, j = (l*NumLaneElts)+NumLaneElts/2; 3539 i != (l+1)*NumLaneElts; i += 2, ++j) { 3540 int BitI = Mask[i]; 3541 int BitI1 = Mask[i+1]; 3542 if (!isUndefOrEqual(BitI, j)) 3543 return false; 3544 if (V2IsSplat) { 3545 if (isUndefOrEqual(BitI1, NumElts)) 3546 return false; 3547 } else { 3548 if (!isUndefOrEqual(BitI1, j+NumElts)) 3549 return false; 3550 } 3551 } 3552 } 3553 return true; 3554} 3555 3556/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 3557/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 3558/// <0, 0, 1, 1> 3559static bool isUNPCKL_v_undef_Mask(ArrayRef<int> Mask, EVT VT, 3560 bool HasAVX2) { 3561 unsigned NumElts = VT.getVectorNumElements(); 3562 3563 assert((VT.is128BitVector() || VT.is256BitVector()) && 3564 "Unsupported vector type for unpckh"); 3565 3566 if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8 && 3567 (!HasAVX2 || (NumElts != 16 && NumElts != 32))) 3568 return false; 3569 3570 // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern 3571 // FIXME: Need a better way to get rid of this, there's no latency difference 3572 // between UNPCKLPD and MOVDDUP, the later should always be checked first and 3573 // the former later. We should also remove the "_undef" special mask. 3574 if (NumElts == 4 && VT.getSizeInBits() == 256) 3575 return false; 3576 3577 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate 3578 // independently on 128-bit lanes. 3579 unsigned NumLanes = VT.getSizeInBits()/128; 3580 unsigned NumLaneElts = NumElts/NumLanes; 3581 3582 for (unsigned l = 0; l != NumLanes; ++l) { 3583 for (unsigned i = l*NumLaneElts, j = l*NumLaneElts; 3584 i != (l+1)*NumLaneElts; 3585 i += 2, ++j) { 3586 int BitI = Mask[i]; 3587 int BitI1 = Mask[i+1]; 3588 3589 if (!isUndefOrEqual(BitI, j)) 3590 return false; 3591 if (!isUndefOrEqual(BitI1, j)) 3592 return false; 3593 } 3594 } 3595 3596 return true; 3597} 3598 3599/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 3600/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 3601/// <2, 2, 3, 3> 3602static bool isUNPCKH_v_undef_Mask(ArrayRef<int> Mask, EVT VT, bool HasAVX2) { 3603 unsigned NumElts = VT.getVectorNumElements(); 3604 3605 assert((VT.is128BitVector() || VT.is256BitVector()) && 3606 "Unsupported vector type for unpckh"); 3607 3608 if (VT.getSizeInBits() == 256 && NumElts != 4 && NumElts != 8 && 3609 (!HasAVX2 || (NumElts != 16 && NumElts != 32))) 3610 return false; 3611 3612 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate 3613 // independently on 128-bit lanes. 3614 unsigned NumLanes = VT.getSizeInBits()/128; 3615 unsigned NumLaneElts = NumElts/NumLanes; 3616 3617 for (unsigned l = 0; l != NumLanes; ++l) { 3618 for (unsigned i = l*NumLaneElts, j = (l*NumLaneElts)+NumLaneElts/2; 3619 i != (l+1)*NumLaneElts; i += 2, ++j) { 3620 int BitI = Mask[i]; 3621 int BitI1 = Mask[i+1]; 3622 if (!isUndefOrEqual(BitI, j)) 3623 return false; 3624 if (!isUndefOrEqual(BitI1, j)) 3625 return false; 3626 } 3627 } 3628 return true; 3629} 3630 3631/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 3632/// specifies a shuffle of elements that is suitable for input to MOVSS, 3633/// MOVSD, and MOVD, i.e. setting the lowest element. 3634static bool isMOVLMask(ArrayRef<int> Mask, EVT VT) { 3635 if (VT.getVectorElementType().getSizeInBits() < 32) 3636 return false; 3637 if (VT.getSizeInBits() == 256) 3638 return false; 3639 3640 unsigned NumElts = VT.getVectorNumElements(); 3641 3642 if (!isUndefOrEqual(Mask[0], NumElts)) 3643 return false; 3644 3645 for (unsigned i = 1; i != NumElts; ++i) 3646 if (!isUndefOrEqual(Mask[i], i)) 3647 return false; 3648 3649 return true; 3650} 3651 3652/// isVPERM2X128Mask - Match 256-bit shuffles where the elements are considered 3653/// as permutations between 128-bit chunks or halves. As an example: this 3654/// shuffle bellow: 3655/// vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15> 3656/// The first half comes from the second half of V1 and the second half from the 3657/// the second half of V2. 3658static bool isVPERM2X128Mask(ArrayRef<int> Mask, EVT VT, bool HasAVX) { 3659 if (!HasAVX || VT.getSizeInBits() != 256) 3660 return false; 3661 3662 // The shuffle result is divided into half A and half B. In total the two 3663 // sources have 4 halves, namely: C, D, E, F. The final values of A and 3664 // B must come from C, D, E or F. 3665 unsigned HalfSize = VT.getVectorNumElements()/2; 3666 bool MatchA = false, MatchB = false; 3667 3668 // Check if A comes from one of C, D, E, F. 3669 for (unsigned Half = 0; Half != 4; ++Half) { 3670 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) { 3671 MatchA = true; 3672 break; 3673 } 3674 } 3675 3676 // Check if B comes from one of C, D, E, F. 3677 for (unsigned Half = 0; Half != 4; ++Half) { 3678 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) { 3679 MatchB = true; 3680 break; 3681 } 3682 } 3683 3684 return MatchA && MatchB; 3685} 3686 3687/// getShuffleVPERM2X128Immediate - Return the appropriate immediate to shuffle 3688/// the specified VECTOR_MASK mask with VPERM2F128/VPERM2I128 instructions. 3689static unsigned getShuffleVPERM2X128Immediate(ShuffleVectorSDNode *SVOp) { 3690 EVT VT = SVOp->getValueType(0); 3691 3692 unsigned HalfSize = VT.getVectorNumElements()/2; 3693 3694 unsigned FstHalf = 0, SndHalf = 0; 3695 for (unsigned i = 0; i < HalfSize; ++i) { 3696 if (SVOp->getMaskElt(i) > 0) { 3697 FstHalf = SVOp->getMaskElt(i)/HalfSize; 3698 break; 3699 } 3700 } 3701 for (unsigned i = HalfSize; i < HalfSize*2; ++i) { 3702 if (SVOp->getMaskElt(i) > 0) { 3703 SndHalf = SVOp->getMaskElt(i)/HalfSize; 3704 break; 3705 } 3706 } 3707 3708 return (FstHalf | (SndHalf << 4)); 3709} 3710 3711/// isVPERMILPMask - Return true if the specified VECTOR_SHUFFLE operand 3712/// specifies a shuffle of elements that is suitable for input to VPERMILPD*. 3713/// Note that VPERMIL mask matching is different depending whether theunderlying 3714/// type is 32 or 64. In the VPERMILPS the high half of the mask should point 3715/// to the same elements of the low, but to the higher half of the source. 3716/// In VPERMILPD the two lanes could be shuffled independently of each other 3717/// with the same restriction that lanes can't be crossed. Also handles PSHUFDY. 3718static bool isVPERMILPMask(ArrayRef<int> Mask, EVT VT, bool HasAVX) { 3719 if (!HasAVX) 3720 return false; 3721 3722 unsigned NumElts = VT.getVectorNumElements(); 3723 // Only match 256-bit with 32/64-bit types 3724 if (VT.getSizeInBits() != 256 || (NumElts != 4 && NumElts != 8)) 3725 return false; 3726 3727 unsigned NumLanes = VT.getSizeInBits()/128; 3728 unsigned LaneSize = NumElts/NumLanes; 3729 for (unsigned l = 0; l != NumElts; l += LaneSize) { 3730 for (unsigned i = 0; i != LaneSize; ++i) { 3731 if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize)) 3732 return false; 3733 if (NumElts != 8 || l == 0) 3734 continue; 3735 // VPERMILPS handling 3736 if (Mask[i] < 0) 3737 continue; 3738 if (!isUndefOrEqual(Mask[i+l], Mask[i]+l)) 3739 return false; 3740 } 3741 } 3742 3743 return true; 3744} 3745 3746/// isCommutedMOVLMask - Returns true if the shuffle mask is except the reverse 3747/// of what x86 movss want. X86 movs requires the lowest element to be lowest 3748/// element of vector 2 and the other elements to come from vector 1 in order. 3749static bool isCommutedMOVLMask(ArrayRef<int> Mask, EVT VT, 3750 bool V2IsSplat = false, bool V2IsUndef = false) { 3751 unsigned NumOps = VT.getVectorNumElements(); 3752 if (VT.getSizeInBits() == 256) 3753 return false; 3754 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 3755 return false; 3756 3757 if (!isUndefOrEqual(Mask[0], 0)) 3758 return false; 3759 3760 for (unsigned i = 1; i != NumOps; ++i) 3761 if (!(isUndefOrEqual(Mask[i], i+NumOps) || 3762 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) || 3763 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps)))) 3764 return false; 3765 3766 return true; 3767} 3768 3769/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3770/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 3771/// Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7> 3772static bool isMOVSHDUPMask(ArrayRef<int> Mask, EVT VT, 3773 const X86Subtarget *Subtarget) { 3774 if (!Subtarget->hasSSE3()) 3775 return false; 3776 3777 unsigned NumElems = VT.getVectorNumElements(); 3778 3779 if ((VT.getSizeInBits() == 128 && NumElems != 4) || 3780 (VT.getSizeInBits() == 256 && NumElems != 8)) 3781 return false; 3782 3783 // "i+1" is the value the indexed mask element must have 3784 for (unsigned i = 0; i != NumElems; i += 2) 3785 if (!isUndefOrEqual(Mask[i], i+1) || 3786 !isUndefOrEqual(Mask[i+1], i+1)) 3787 return false; 3788 3789 return true; 3790} 3791 3792/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3793/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 3794/// Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6> 3795static bool isMOVSLDUPMask(ArrayRef<int> Mask, EVT VT, 3796 const X86Subtarget *Subtarget) { 3797 if (!Subtarget->hasSSE3()) 3798 return false; 3799 3800 unsigned NumElems = VT.getVectorNumElements(); 3801 3802 if ((VT.getSizeInBits() == 128 && NumElems != 4) || 3803 (VT.getSizeInBits() == 256 && NumElems != 8)) 3804 return false; 3805 3806 // "i" is the value the indexed mask element must have 3807 for (unsigned i = 0; i != NumElems; i += 2) 3808 if (!isUndefOrEqual(Mask[i], i) || 3809 !isUndefOrEqual(Mask[i+1], i)) 3810 return false; 3811 3812 return true; 3813} 3814 3815/// isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand 3816/// specifies a shuffle of elements that is suitable for input to 256-bit 3817/// version of MOVDDUP. 3818static bool isMOVDDUPYMask(ArrayRef<int> Mask, EVT VT, bool HasAVX) { 3819 unsigned NumElts = VT.getVectorNumElements(); 3820 3821 if (!HasAVX || VT.getSizeInBits() != 256 || NumElts != 4) 3822 return false; 3823 3824 for (unsigned i = 0; i != NumElts/2; ++i) 3825 if (!isUndefOrEqual(Mask[i], 0)) 3826 return false; 3827 for (unsigned i = NumElts/2; i != NumElts; ++i) 3828 if (!isUndefOrEqual(Mask[i], NumElts/2)) 3829 return false; 3830 return true; 3831} 3832 3833/// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand 3834/// specifies a shuffle of elements that is suitable for input to 128-bit 3835/// version of MOVDDUP. 3836static bool isMOVDDUPMask(ArrayRef<int> Mask, EVT VT) { 3837 if (VT.getSizeInBits() != 128) 3838 return false; 3839 3840 unsigned e = VT.getVectorNumElements() / 2; 3841 for (unsigned i = 0; i != e; ++i) 3842 if (!isUndefOrEqual(Mask[i], i)) 3843 return false; 3844 for (unsigned i = 0; i != e; ++i) 3845 if (!isUndefOrEqual(Mask[e+i], i)) 3846 return false; 3847 return true; 3848} 3849 3850/// isVEXTRACTF128Index - Return true if the specified 3851/// EXTRACT_SUBVECTOR operand specifies a vector extract that is 3852/// suitable for input to VEXTRACTF128. 3853bool X86::isVEXTRACTF128Index(SDNode *N) { 3854 if (!isa<ConstantSDNode>(N->getOperand(1).getNode())) 3855 return false; 3856 3857 // The index should be aligned on a 128-bit boundary. 3858 uint64_t Index = 3859 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue(); 3860 3861 unsigned VL = N->getValueType(0).getVectorNumElements(); 3862 unsigned VBits = N->getValueType(0).getSizeInBits(); 3863 unsigned ElSize = VBits / VL; 3864 bool Result = (Index * ElSize) % 128 == 0; 3865 3866 return Result; 3867} 3868 3869/// isVINSERTF128Index - Return true if the specified INSERT_SUBVECTOR 3870/// operand specifies a subvector insert that is suitable for input to 3871/// VINSERTF128. 3872bool X86::isVINSERTF128Index(SDNode *N) { 3873 if (!isa<ConstantSDNode>(N->getOperand(2).getNode())) 3874 return false; 3875 3876 // The index should be aligned on a 128-bit boundary. 3877 uint64_t Index = 3878 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue(); 3879 3880 unsigned VL = N->getValueType(0).getVectorNumElements(); 3881 unsigned VBits = N->getValueType(0).getSizeInBits(); 3882 unsigned ElSize = VBits / VL; 3883 bool Result = (Index * ElSize) % 128 == 0; 3884 3885 return Result; 3886} 3887 3888/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 3889/// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions. 3890/// Handles 128-bit and 256-bit. 3891static unsigned getShuffleSHUFImmediate(ShuffleVectorSDNode *N) { 3892 EVT VT = N->getValueType(0); 3893 3894 assert((VT.is128BitVector() || VT.is256BitVector()) && 3895 "Unsupported vector type for PSHUF/SHUFP"); 3896 3897 // Handle 128 and 256-bit vector lengths. AVX defines PSHUF/SHUFP to operate 3898 // independently on 128-bit lanes. 3899 unsigned NumElts = VT.getVectorNumElements(); 3900 unsigned NumLanes = VT.getSizeInBits()/128; 3901 unsigned NumLaneElts = NumElts/NumLanes; 3902 3903 assert((NumLaneElts == 2 || NumLaneElts == 4) && 3904 "Only supports 2 or 4 elements per lane"); 3905 3906 unsigned Shift = (NumLaneElts == 4) ? 1 : 0; 3907 unsigned Mask = 0; 3908 for (unsigned i = 0; i != NumElts; ++i) { 3909 int Elt = N->getMaskElt(i); 3910 if (Elt < 0) continue; 3911 Elt &= NumLaneElts - 1; 3912 unsigned ShAmt = (i << Shift) % 8; 3913 Mask |= Elt << ShAmt; 3914 } 3915 3916 return Mask; 3917} 3918 3919/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 3920/// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction. 3921static unsigned getShufflePSHUFHWImmediate(ShuffleVectorSDNode *N) { 3922 EVT VT = N->getValueType(0); 3923 3924 assert((VT == MVT::v8i16 || VT == MVT::v16i16) && 3925 "Unsupported vector type for PSHUFHW"); 3926 3927 unsigned NumElts = VT.getVectorNumElements(); 3928 3929 unsigned Mask = 0; 3930 for (unsigned l = 0; l != NumElts; l += 8) { 3931 // 8 nodes per lane, but we only care about the last 4. 3932 for (unsigned i = 0; i < 4; ++i) { 3933 int Elt = N->getMaskElt(l+i+4); 3934 if (Elt < 0) continue; 3935 Elt &= 0x3; // only 2-bits. 3936 Mask |= Elt << (i * 2); 3937 } 3938 } 3939 3940 return Mask; 3941} 3942 3943/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 3944/// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction. 3945static unsigned getShufflePSHUFLWImmediate(ShuffleVectorSDNode *N) { 3946 EVT VT = N->getValueType(0); 3947 3948 assert((VT == MVT::v8i16 || VT == MVT::v16i16) && 3949 "Unsupported vector type for PSHUFHW"); 3950 3951 unsigned NumElts = VT.getVectorNumElements(); 3952 3953 unsigned Mask = 0; 3954 for (unsigned l = 0; l != NumElts; l += 8) { 3955 // 8 nodes per lane, but we only care about the first 4. 3956 for (unsigned i = 0; i < 4; ++i) { 3957 int Elt = N->getMaskElt(l+i); 3958 if (Elt < 0) continue; 3959 Elt &= 0x3; // only 2-bits 3960 Mask |= Elt << (i * 2); 3961 } 3962 } 3963 3964 return Mask; 3965} 3966 3967/// getShufflePALIGNRImmediate - Return the appropriate immediate to shuffle 3968/// the specified VECTOR_SHUFFLE mask with the PALIGNR instruction. 3969static unsigned getShufflePALIGNRImmediate(ShuffleVectorSDNode *SVOp) { 3970 EVT VT = SVOp->getValueType(0); 3971 unsigned EltSize = VT.getVectorElementType().getSizeInBits() >> 3; 3972 3973 unsigned NumElts = VT.getVectorNumElements(); 3974 unsigned NumLanes = VT.getSizeInBits()/128; 3975 unsigned NumLaneElts = NumElts/NumLanes; 3976 3977 int Val = 0; 3978 unsigned i; 3979 for (i = 0; i != NumElts; ++i) { 3980 Val = SVOp->getMaskElt(i); 3981 if (Val >= 0) 3982 break; 3983 } 3984 if (Val >= (int)NumElts) 3985 Val -= NumElts - NumLaneElts; 3986 3987 assert(Val - i > 0 && "PALIGNR imm should be positive"); 3988 return (Val - i) * EltSize; 3989} 3990 3991/// getExtractVEXTRACTF128Immediate - Return the appropriate immediate 3992/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128 3993/// instructions. 3994unsigned X86::getExtractVEXTRACTF128Immediate(SDNode *N) { 3995 if (!isa<ConstantSDNode>(N->getOperand(1).getNode())) 3996 llvm_unreachable("Illegal extract subvector for VEXTRACTF128"); 3997 3998 uint64_t Index = 3999 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue(); 4000 4001 EVT VecVT = N->getOperand(0).getValueType(); 4002 EVT ElVT = VecVT.getVectorElementType(); 4003 4004 unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits(); 4005 return Index / NumElemsPerChunk; 4006} 4007 4008/// getInsertVINSERTF128Immediate - Return the appropriate immediate 4009/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128 4010/// instructions. 4011unsigned X86::getInsertVINSERTF128Immediate(SDNode *N) { 4012 if (!isa<ConstantSDNode>(N->getOperand(2).getNode())) 4013 llvm_unreachable("Illegal insert subvector for VINSERTF128"); 4014 4015 uint64_t Index = 4016 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue(); 4017 4018 EVT VecVT = N->getValueType(0); 4019 EVT ElVT = VecVT.getVectorElementType(); 4020 4021 unsigned NumElemsPerChunk = 128 / ElVT.getSizeInBits(); 4022 return Index / NumElemsPerChunk; 4023} 4024 4025/// getShuffleCLImmediate - Return the appropriate immediate to shuffle 4026/// the specified VECTOR_SHUFFLE mask with VPERMQ and VPERMPD instructions. 4027/// Handles 256-bit. 4028static unsigned getShuffleCLImmediate(ShuffleVectorSDNode *N) { 4029 EVT VT = N->getValueType(0); 4030 4031 unsigned NumElts = VT.getVectorNumElements(); 4032 4033 assert((VT.is256BitVector() && NumElts == 4) && 4034 "Unsupported vector type for VPERMQ/VPERMPD"); 4035 4036 unsigned Mask = 0; 4037 for (unsigned i = 0; i != NumElts; ++i) { 4038 int Elt = N->getMaskElt(i); 4039 if (Elt < 0) 4040 continue; 4041 Mask |= Elt << (i*2); 4042 } 4043 4044 return Mask; 4045} 4046/// isZeroNode - Returns true if Elt is a constant zero or a floating point 4047/// constant +0.0. 4048bool X86::isZeroNode(SDValue Elt) { 4049 return ((isa<ConstantSDNode>(Elt) && 4050 cast<ConstantSDNode>(Elt)->isNullValue()) || 4051 (isa<ConstantFPSDNode>(Elt) && 4052 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 4053} 4054 4055/// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in 4056/// their permute mask. 4057static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp, 4058 SelectionDAG &DAG) { 4059 EVT VT = SVOp->getValueType(0); 4060 unsigned NumElems = VT.getVectorNumElements(); 4061 SmallVector<int, 8> MaskVec; 4062 4063 for (unsigned i = 0; i != NumElems; ++i) { 4064 int idx = SVOp->getMaskElt(i); 4065 if (idx < 0) 4066 MaskVec.push_back(idx); 4067 else if (idx < (int)NumElems) 4068 MaskVec.push_back(idx + NumElems); 4069 else 4070 MaskVec.push_back(idx - NumElems); 4071 } 4072 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1), 4073 SVOp->getOperand(0), &MaskVec[0]); 4074} 4075 4076/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 4077/// match movhlps. The lower half elements should come from upper half of 4078/// V1 (and in order), and the upper half elements should come from the upper 4079/// half of V2 (and in order). 4080static bool ShouldXformToMOVHLPS(ArrayRef<int> Mask, EVT VT) { 4081 if (VT.getSizeInBits() != 128) 4082 return false; 4083 if (VT.getVectorNumElements() != 4) 4084 return false; 4085 for (unsigned i = 0, e = 2; i != e; ++i) 4086 if (!isUndefOrEqual(Mask[i], i+2)) 4087 return false; 4088 for (unsigned i = 2; i != 4; ++i) 4089 if (!isUndefOrEqual(Mask[i], i+4)) 4090 return false; 4091 return true; 4092} 4093 4094/// isScalarLoadToVector - Returns true if the node is a scalar load that 4095/// is promoted to a vector. It also returns the LoadSDNode by reference if 4096/// required. 4097static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 4098 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR) 4099 return false; 4100 N = N->getOperand(0).getNode(); 4101 if (!ISD::isNON_EXTLoad(N)) 4102 return false; 4103 if (LD) 4104 *LD = cast<LoadSDNode>(N); 4105 return true; 4106} 4107 4108// Test whether the given value is a vector value which will be legalized 4109// into a load. 4110static bool WillBeConstantPoolLoad(SDNode *N) { 4111 if (N->getOpcode() != ISD::BUILD_VECTOR) 4112 return false; 4113 4114 // Check for any non-constant elements. 4115 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 4116 switch (N->getOperand(i).getNode()->getOpcode()) { 4117 case ISD::UNDEF: 4118 case ISD::ConstantFP: 4119 case ISD::Constant: 4120 break; 4121 default: 4122 return false; 4123 } 4124 4125 // Vectors of all-zeros and all-ones are materialized with special 4126 // instructions rather than being loaded. 4127 return !ISD::isBuildVectorAllZeros(N) && 4128 !ISD::isBuildVectorAllOnes(N); 4129} 4130 4131/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 4132/// match movlp{s|d}. The lower half elements should come from lower half of 4133/// V1 (and in order), and the upper half elements should come from the upper 4134/// half of V2 (and in order). And since V1 will become the source of the 4135/// MOVLP, it must be either a vector load or a scalar load to vector. 4136static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, 4137 ArrayRef<int> Mask, EVT VT) { 4138 if (VT.getSizeInBits() != 128) 4139 return false; 4140 4141 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 4142 return false; 4143 // Is V2 is a vector load, don't do this transformation. We will try to use 4144 // load folding shufps op. 4145 if (ISD::isNON_EXTLoad(V2) || WillBeConstantPoolLoad(V2)) 4146 return false; 4147 4148 unsigned NumElems = VT.getVectorNumElements(); 4149 4150 if (NumElems != 2 && NumElems != 4) 4151 return false; 4152 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 4153 if (!isUndefOrEqual(Mask[i], i)) 4154 return false; 4155 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i) 4156 if (!isUndefOrEqual(Mask[i], i+NumElems)) 4157 return false; 4158 return true; 4159} 4160 4161/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 4162/// all the same. 4163static bool isSplatVector(SDNode *N) { 4164 if (N->getOpcode() != ISD::BUILD_VECTOR) 4165 return false; 4166 4167 SDValue SplatValue = N->getOperand(0); 4168 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 4169 if (N->getOperand(i) != SplatValue) 4170 return false; 4171 return true; 4172} 4173 4174/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 4175/// to an zero vector. 4176/// FIXME: move to dag combiner / method on ShuffleVectorSDNode 4177static bool isZeroShuffle(ShuffleVectorSDNode *N) { 4178 SDValue V1 = N->getOperand(0); 4179 SDValue V2 = N->getOperand(1); 4180 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 4181 for (unsigned i = 0; i != NumElems; ++i) { 4182 int Idx = N->getMaskElt(i); 4183 if (Idx >= (int)NumElems) { 4184 unsigned Opc = V2.getOpcode(); 4185 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) 4186 continue; 4187 if (Opc != ISD::BUILD_VECTOR || 4188 !X86::isZeroNode(V2.getOperand(Idx-NumElems))) 4189 return false; 4190 } else if (Idx >= 0) { 4191 unsigned Opc = V1.getOpcode(); 4192 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) 4193 continue; 4194 if (Opc != ISD::BUILD_VECTOR || 4195 !X86::isZeroNode(V1.getOperand(Idx))) 4196 return false; 4197 } 4198 } 4199 return true; 4200} 4201 4202/// getZeroVector - Returns a vector of specified type with all zero elements. 4203/// 4204static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget, 4205 SelectionDAG &DAG, DebugLoc dl) { 4206 assert(VT.isVector() && "Expected a vector type"); 4207 unsigned Size = VT.getSizeInBits(); 4208 4209 // Always build SSE zero vectors as <4 x i32> bitcasted 4210 // to their dest type. This ensures they get CSE'd. 4211 SDValue Vec; 4212 if (Size == 128) { // SSE 4213 if (Subtarget->hasSSE2()) { // SSE2 4214 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 4215 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 4216 } else { // SSE1 4217 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 4218 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); 4219 } 4220 } else if (Size == 256) { // AVX 4221 if (Subtarget->hasAVX2()) { // AVX2 4222 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 4223 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; 4224 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops, 8); 4225 } else { 4226 // 256-bit logic and arithmetic instructions in AVX are all 4227 // floating-point, no support for integer ops. Emit fp zeroed vectors. 4228 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 4229 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; 4230 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops, 8); 4231 } 4232 } else 4233 llvm_unreachable("Unexpected vector type"); 4234 4235 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); 4236} 4237 4238/// getOnesVector - Returns a vector of specified type with all bits set. 4239/// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with 4240/// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. 4241/// Then bitcast to their original type, ensuring they get CSE'd. 4242static SDValue getOnesVector(EVT VT, bool HasAVX2, SelectionDAG &DAG, 4243 DebugLoc dl) { 4244 assert(VT.isVector() && "Expected a vector type"); 4245 unsigned Size = VT.getSizeInBits(); 4246 4247 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 4248 SDValue Vec; 4249 if (Size == 256) { 4250 if (HasAVX2) { // AVX2 4251 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; 4252 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops, 8); 4253 } else { // AVX 4254 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 4255 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); 4256 } 4257 } else if (Size == 128) { 4258 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 4259 } else 4260 llvm_unreachable("Unexpected vector type"); 4261 4262 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); 4263} 4264 4265/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 4266/// that point to V2 points to its first element. 4267static void NormalizeMask(SmallVectorImpl<int> &Mask, unsigned NumElems) { 4268 for (unsigned i = 0; i != NumElems; ++i) { 4269 if (Mask[i] > (int)NumElems) { 4270 Mask[i] = NumElems; 4271 } 4272 } 4273} 4274 4275/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 4276/// operation of specified width. 4277static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 4278 SDValue V2) { 4279 unsigned NumElems = VT.getVectorNumElements(); 4280 SmallVector<int, 8> Mask; 4281 Mask.push_back(NumElems); 4282 for (unsigned i = 1; i != NumElems; ++i) 4283 Mask.push_back(i); 4284 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 4285} 4286 4287/// getUnpackl - Returns a vector_shuffle node for an unpackl operation. 4288static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 4289 SDValue V2) { 4290 unsigned NumElems = VT.getVectorNumElements(); 4291 SmallVector<int, 8> Mask; 4292 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 4293 Mask.push_back(i); 4294 Mask.push_back(i + NumElems); 4295 } 4296 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 4297} 4298 4299/// getUnpackh - Returns a vector_shuffle node for an unpackh operation. 4300static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 4301 SDValue V2) { 4302 unsigned NumElems = VT.getVectorNumElements(); 4303 SmallVector<int, 8> Mask; 4304 for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) { 4305 Mask.push_back(i + Half); 4306 Mask.push_back(i + NumElems + Half); 4307 } 4308 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 4309} 4310 4311// PromoteSplati8i16 - All i16 and i8 vector types can't be used directly by 4312// a generic shuffle instruction because the target has no such instructions. 4313// Generate shuffles which repeat i16 and i8 several times until they can be 4314// represented by v4f32 and then be manipulated by target suported shuffles. 4315static SDValue PromoteSplati8i16(SDValue V, SelectionDAG &DAG, int &EltNo) { 4316 EVT VT = V.getValueType(); 4317 int NumElems = VT.getVectorNumElements(); 4318 DebugLoc dl = V.getDebugLoc(); 4319 4320 while (NumElems > 4) { 4321 if (EltNo < NumElems/2) { 4322 V = getUnpackl(DAG, dl, VT, V, V); 4323 } else { 4324 V = getUnpackh(DAG, dl, VT, V, V); 4325 EltNo -= NumElems/2; 4326 } 4327 NumElems >>= 1; 4328 } 4329 return V; 4330} 4331 4332/// getLegalSplat - Generate a legal splat with supported x86 shuffles 4333static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) { 4334 EVT VT = V.getValueType(); 4335 DebugLoc dl = V.getDebugLoc(); 4336 unsigned Size = VT.getSizeInBits(); 4337 4338 if (Size == 128) { 4339 V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V); 4340 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo }; 4341 V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32), 4342 &SplatMask[0]); 4343 } else if (Size == 256) { 4344 // To use VPERMILPS to splat scalars, the second half of indicies must 4345 // refer to the higher part, which is a duplication of the lower one, 4346 // because VPERMILPS can only handle in-lane permutations. 4347 int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo, 4348 EltNo+4, EltNo+4, EltNo+4, EltNo+4 }; 4349 4350 V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V); 4351 V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32), 4352 &SplatMask[0]); 4353 } else 4354 llvm_unreachable("Vector size not supported"); 4355 4356 return DAG.getNode(ISD::BITCAST, dl, VT, V); 4357} 4358 4359/// PromoteSplat - Splat is promoted to target supported vector shuffles. 4360static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) { 4361 EVT SrcVT = SV->getValueType(0); 4362 SDValue V1 = SV->getOperand(0); 4363 DebugLoc dl = SV->getDebugLoc(); 4364 4365 int EltNo = SV->getSplatIndex(); 4366 int NumElems = SrcVT.getVectorNumElements(); 4367 unsigned Size = SrcVT.getSizeInBits(); 4368 4369 assert(((Size == 128 && NumElems > 4) || Size == 256) && 4370 "Unknown how to promote splat for type"); 4371 4372 // Extract the 128-bit part containing the splat element and update 4373 // the splat element index when it refers to the higher register. 4374 if (Size == 256) { 4375 V1 = Extract128BitVector(V1, EltNo, DAG, dl); 4376 if (EltNo >= NumElems/2) 4377 EltNo -= NumElems/2; 4378 } 4379 4380 // All i16 and i8 vector types can't be used directly by a generic shuffle 4381 // instruction because the target has no such instruction. Generate shuffles 4382 // which repeat i16 and i8 several times until they fit in i32, and then can 4383 // be manipulated by target suported shuffles. 4384 EVT EltVT = SrcVT.getVectorElementType(); 4385 if (EltVT == MVT::i8 || EltVT == MVT::i16) 4386 V1 = PromoteSplati8i16(V1, DAG, EltNo); 4387 4388 // Recreate the 256-bit vector and place the same 128-bit vector 4389 // into the low and high part. This is necessary because we want 4390 // to use VPERM* to shuffle the vectors 4391 if (Size == 256) { 4392 V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, V1, V1); 4393 } 4394 4395 return getLegalSplat(DAG, V1, EltNo); 4396} 4397 4398/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 4399/// vector of zero or undef vector. This produces a shuffle where the low 4400/// element of V2 is swizzled into the zero/undef vector, landing at element 4401/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 4402static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 4403 bool IsZero, 4404 const X86Subtarget *Subtarget, 4405 SelectionDAG &DAG) { 4406 EVT VT = V2.getValueType(); 4407 SDValue V1 = IsZero 4408 ? getZeroVector(VT, Subtarget, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT); 4409 unsigned NumElems = VT.getVectorNumElements(); 4410 SmallVector<int, 16> MaskVec; 4411 for (unsigned i = 0; i != NumElems; ++i) 4412 // If this is the insertion idx, put the low elt of V2 here. 4413 MaskVec.push_back(i == Idx ? NumElems : i); 4414 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]); 4415} 4416 4417/// getTargetShuffleMask - Calculates the shuffle mask corresponding to the 4418/// target specific opcode. Returns true if the Mask could be calculated. 4419/// Sets IsUnary to true if only uses one source. 4420static bool getTargetShuffleMask(SDNode *N, MVT VT, 4421 SmallVectorImpl<int> &Mask, bool &IsUnary) { 4422 unsigned NumElems = VT.getVectorNumElements(); 4423 SDValue ImmN; 4424 4425 IsUnary = false; 4426 switch(N->getOpcode()) { 4427 case X86ISD::SHUFP: 4428 ImmN = N->getOperand(N->getNumOperands()-1); 4429 DecodeSHUFPMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4430 break; 4431 case X86ISD::UNPCKH: 4432 DecodeUNPCKHMask(VT, Mask); 4433 break; 4434 case X86ISD::UNPCKL: 4435 DecodeUNPCKLMask(VT, Mask); 4436 break; 4437 case X86ISD::MOVHLPS: 4438 DecodeMOVHLPSMask(NumElems, Mask); 4439 break; 4440 case X86ISD::MOVLHPS: 4441 DecodeMOVLHPSMask(NumElems, Mask); 4442 break; 4443 case X86ISD::PSHUFD: 4444 case X86ISD::VPERMILP: 4445 ImmN = N->getOperand(N->getNumOperands()-1); 4446 DecodePSHUFMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4447 IsUnary = true; 4448 break; 4449 case X86ISD::PSHUFHW: 4450 ImmN = N->getOperand(N->getNumOperands()-1); 4451 DecodePSHUFHWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4452 IsUnary = true; 4453 break; 4454 case X86ISD::PSHUFLW: 4455 ImmN = N->getOperand(N->getNumOperands()-1); 4456 DecodePSHUFLWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4457 IsUnary = true; 4458 break; 4459 case X86ISD::VPERMI: 4460 ImmN = N->getOperand(N->getNumOperands()-1); 4461 DecodeVPERMMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4462 IsUnary = true; 4463 break; 4464 case X86ISD::MOVSS: 4465 case X86ISD::MOVSD: { 4466 // The index 0 always comes from the first element of the second source, 4467 // this is why MOVSS and MOVSD are used in the first place. The other 4468 // elements come from the other positions of the first source vector 4469 Mask.push_back(NumElems); 4470 for (unsigned i = 1; i != NumElems; ++i) { 4471 Mask.push_back(i); 4472 } 4473 break; 4474 } 4475 case X86ISD::VPERM2X128: 4476 ImmN = N->getOperand(N->getNumOperands()-1); 4477 DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask); 4478 if (Mask.empty()) return false; 4479 break; 4480 case X86ISD::MOVDDUP: 4481 case X86ISD::MOVLHPD: 4482 case X86ISD::MOVLPD: 4483 case X86ISD::MOVLPS: 4484 case X86ISD::MOVSHDUP: 4485 case X86ISD::MOVSLDUP: 4486 case X86ISD::PALIGN: 4487 // Not yet implemented 4488 return false; 4489 default: llvm_unreachable("unknown target shuffle node"); 4490 } 4491 4492 return true; 4493} 4494 4495/// getShuffleScalarElt - Returns the scalar element that will make up the ith 4496/// element of the result of the vector shuffle. 4497static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG, 4498 unsigned Depth) { 4499 if (Depth == 6) 4500 return SDValue(); // Limit search depth. 4501 4502 SDValue V = SDValue(N, 0); 4503 EVT VT = V.getValueType(); 4504 unsigned Opcode = V.getOpcode(); 4505 4506 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars. 4507 if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) { 4508 int Elt = SV->getMaskElt(Index); 4509 4510 if (Elt < 0) 4511 return DAG.getUNDEF(VT.getVectorElementType()); 4512 4513 unsigned NumElems = VT.getVectorNumElements(); 4514 SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0) 4515 : SV->getOperand(1); 4516 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1); 4517 } 4518 4519 // Recurse into target specific vector shuffles to find scalars. 4520 if (isTargetShuffle(Opcode)) { 4521 MVT ShufVT = V.getValueType().getSimpleVT(); 4522 unsigned NumElems = ShufVT.getVectorNumElements(); 4523 SmallVector<int, 16> ShuffleMask; 4524 SDValue ImmN; 4525 bool IsUnary; 4526 4527 if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary)) 4528 return SDValue(); 4529 4530 int Elt = ShuffleMask[Index]; 4531 if (Elt < 0) 4532 return DAG.getUNDEF(ShufVT.getVectorElementType()); 4533 4534 SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0) 4535 : N->getOperand(1); 4536 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, 4537 Depth+1); 4538 } 4539 4540 // Actual nodes that may contain scalar elements 4541 if (Opcode == ISD::BITCAST) { 4542 V = V.getOperand(0); 4543 EVT SrcVT = V.getValueType(); 4544 unsigned NumElems = VT.getVectorNumElements(); 4545 4546 if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems) 4547 return SDValue(); 4548 } 4549 4550 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) 4551 return (Index == 0) ? V.getOperand(0) 4552 : DAG.getUNDEF(VT.getVectorElementType()); 4553 4554 if (V.getOpcode() == ISD::BUILD_VECTOR) 4555 return V.getOperand(Index); 4556 4557 return SDValue(); 4558} 4559 4560/// getNumOfConsecutiveZeros - Return the number of elements of a vector 4561/// shuffle operation which come from a consecutively from a zero. The 4562/// search can start in two different directions, from left or right. 4563static 4564unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp, unsigned NumElems, 4565 bool ZerosFromLeft, SelectionDAG &DAG) { 4566 unsigned i; 4567 for (i = 0; i != NumElems; ++i) { 4568 unsigned Index = ZerosFromLeft ? i : NumElems-i-1; 4569 SDValue Elt = getShuffleScalarElt(SVOp, Index, DAG, 0); 4570 if (!(Elt.getNode() && 4571 (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt)))) 4572 break; 4573 } 4574 4575 return i; 4576} 4577 4578/// isShuffleMaskConsecutive - Check if the shuffle mask indicies [MaskI, MaskE) 4579/// correspond consecutively to elements from one of the vector operands, 4580/// starting from its index OpIdx. Also tell OpNum which source vector operand. 4581static 4582bool isShuffleMaskConsecutive(ShuffleVectorSDNode *SVOp, 4583 unsigned MaskI, unsigned MaskE, unsigned OpIdx, 4584 unsigned NumElems, unsigned &OpNum) { 4585 bool SeenV1 = false; 4586 bool SeenV2 = false; 4587 4588 for (unsigned i = MaskI; i != MaskE; ++i, ++OpIdx) { 4589 int Idx = SVOp->getMaskElt(i); 4590 // Ignore undef indicies 4591 if (Idx < 0) 4592 continue; 4593 4594 if (Idx < (int)NumElems) 4595 SeenV1 = true; 4596 else 4597 SeenV2 = true; 4598 4599 // Only accept consecutive elements from the same vector 4600 if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2)) 4601 return false; 4602 } 4603 4604 OpNum = SeenV1 ? 0 : 1; 4605 return true; 4606} 4607 4608/// isVectorShiftRight - Returns true if the shuffle can be implemented as a 4609/// logical left shift of a vector. 4610static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4611 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4612 unsigned NumElems = SVOp->getValueType(0).getVectorNumElements(); 4613 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, 4614 false /* check zeros from right */, DAG); 4615 unsigned OpSrc; 4616 4617 if (!NumZeros) 4618 return false; 4619 4620 // Considering the elements in the mask that are not consecutive zeros, 4621 // check if they consecutively come from only one of the source vectors. 4622 // 4623 // V1 = {X, A, B, C} 0 4624 // \ \ \ / 4625 // vector_shuffle V1, V2 <1, 2, 3, X> 4626 // 4627 if (!isShuffleMaskConsecutive(SVOp, 4628 0, // Mask Start Index 4629 NumElems-NumZeros, // Mask End Index(exclusive) 4630 NumZeros, // Where to start looking in the src vector 4631 NumElems, // Number of elements in vector 4632 OpSrc)) // Which source operand ? 4633 return false; 4634 4635 isLeft = false; 4636 ShAmt = NumZeros; 4637 ShVal = SVOp->getOperand(OpSrc); 4638 return true; 4639} 4640 4641/// isVectorShiftLeft - Returns true if the shuffle can be implemented as a 4642/// logical left shift of a vector. 4643static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4644 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4645 unsigned NumElems = SVOp->getValueType(0).getVectorNumElements(); 4646 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, 4647 true /* check zeros from left */, DAG); 4648 unsigned OpSrc; 4649 4650 if (!NumZeros) 4651 return false; 4652 4653 // Considering the elements in the mask that are not consecutive zeros, 4654 // check if they consecutively come from only one of the source vectors. 4655 // 4656 // 0 { A, B, X, X } = V2 4657 // / \ / / 4658 // vector_shuffle V1, V2 <X, X, 4, 5> 4659 // 4660 if (!isShuffleMaskConsecutive(SVOp, 4661 NumZeros, // Mask Start Index 4662 NumElems, // Mask End Index(exclusive) 4663 0, // Where to start looking in the src vector 4664 NumElems, // Number of elements in vector 4665 OpSrc)) // Which source operand ? 4666 return false; 4667 4668 isLeft = true; 4669 ShAmt = NumZeros; 4670 ShVal = SVOp->getOperand(OpSrc); 4671 return true; 4672} 4673 4674/// isVectorShift - Returns true if the shuffle can be implemented as a 4675/// logical left or right shift of a vector. 4676static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 4677 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 4678 // Although the logic below support any bitwidth size, there are no 4679 // shift instructions which handle more than 128-bit vectors. 4680 if (SVOp->getValueType(0).getSizeInBits() > 128) 4681 return false; 4682 4683 if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) || 4684 isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt)) 4685 return true; 4686 4687 return false; 4688} 4689 4690/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 4691/// 4692static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 4693 unsigned NumNonZero, unsigned NumZero, 4694 SelectionDAG &DAG, 4695 const X86Subtarget* Subtarget, 4696 const TargetLowering &TLI) { 4697 if (NumNonZero > 8) 4698 return SDValue(); 4699 4700 DebugLoc dl = Op.getDebugLoc(); 4701 SDValue V(0, 0); 4702 bool First = true; 4703 for (unsigned i = 0; i < 16; ++i) { 4704 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 4705 if (ThisIsNonZero && First) { 4706 if (NumZero) 4707 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); 4708 else 4709 V = DAG.getUNDEF(MVT::v8i16); 4710 First = false; 4711 } 4712 4713 if ((i & 1) != 0) { 4714 SDValue ThisElt(0, 0), LastElt(0, 0); 4715 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 4716 if (LastIsNonZero) { 4717 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, 4718 MVT::i16, Op.getOperand(i-1)); 4719 } 4720 if (ThisIsNonZero) { 4721 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); 4722 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, 4723 ThisElt, DAG.getConstant(8, MVT::i8)); 4724 if (LastIsNonZero) 4725 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); 4726 } else 4727 ThisElt = LastElt; 4728 4729 if (ThisElt.getNode()) 4730 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, 4731 DAG.getIntPtrConstant(i/2)); 4732 } 4733 } 4734 4735 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); 4736} 4737 4738/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 4739/// 4740static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 4741 unsigned NumNonZero, unsigned NumZero, 4742 SelectionDAG &DAG, 4743 const X86Subtarget* Subtarget, 4744 const TargetLowering &TLI) { 4745 if (NumNonZero > 4) 4746 return SDValue(); 4747 4748 DebugLoc dl = Op.getDebugLoc(); 4749 SDValue V(0, 0); 4750 bool First = true; 4751 for (unsigned i = 0; i < 8; ++i) { 4752 bool isNonZero = (NonZeros & (1 << i)) != 0; 4753 if (isNonZero) { 4754 if (First) { 4755 if (NumZero) 4756 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); 4757 else 4758 V = DAG.getUNDEF(MVT::v8i16); 4759 First = false; 4760 } 4761 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 4762 MVT::v8i16, V, Op.getOperand(i), 4763 DAG.getIntPtrConstant(i)); 4764 } 4765 } 4766 4767 return V; 4768} 4769 4770/// getVShift - Return a vector logical shift node. 4771/// 4772static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, 4773 unsigned NumBits, SelectionDAG &DAG, 4774 const TargetLowering &TLI, DebugLoc dl) { 4775 assert(VT.getSizeInBits() == 128 && "Unknown type for VShift"); 4776 EVT ShVT = MVT::v2i64; 4777 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ; 4778 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); 4779 return DAG.getNode(ISD::BITCAST, dl, VT, 4780 DAG.getNode(Opc, dl, ShVT, SrcOp, 4781 DAG.getConstant(NumBits, 4782 TLI.getShiftAmountTy(SrcOp.getValueType())))); 4783} 4784 4785SDValue 4786X86TargetLowering::LowerAsSplatVectorLoad(SDValue SrcOp, EVT VT, DebugLoc dl, 4787 SelectionDAG &DAG) const { 4788 4789 // Check if the scalar load can be widened into a vector load. And if 4790 // the address is "base + cst" see if the cst can be "absorbed" into 4791 // the shuffle mask. 4792 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) { 4793 SDValue Ptr = LD->getBasePtr(); 4794 if (!ISD::isNormalLoad(LD) || LD->isVolatile()) 4795 return SDValue(); 4796 EVT PVT = LD->getValueType(0); 4797 if (PVT != MVT::i32 && PVT != MVT::f32) 4798 return SDValue(); 4799 4800 int FI = -1; 4801 int64_t Offset = 0; 4802 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) { 4803 FI = FINode->getIndex(); 4804 Offset = 0; 4805 } else if (DAG.isBaseWithConstantOffset(Ptr) && 4806 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 4807 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4808 Offset = Ptr.getConstantOperandVal(1); 4809 Ptr = Ptr.getOperand(0); 4810 } else { 4811 return SDValue(); 4812 } 4813 4814 // FIXME: 256-bit vector instructions don't require a strict alignment, 4815 // improve this code to support it better. 4816 unsigned RequiredAlign = VT.getSizeInBits()/8; 4817 SDValue Chain = LD->getChain(); 4818 // Make sure the stack object alignment is at least 16 or 32. 4819 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4820 if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) { 4821 if (MFI->isFixedObjectIndex(FI)) { 4822 // Can't change the alignment. FIXME: It's possible to compute 4823 // the exact stack offset and reference FI + adjust offset instead. 4824 // If someone *really* cares about this. That's the way to implement it. 4825 return SDValue(); 4826 } else { 4827 MFI->setObjectAlignment(FI, RequiredAlign); 4828 } 4829 } 4830 4831 // (Offset % 16 or 32) must be multiple of 4. Then address is then 4832 // Ptr + (Offset & ~15). 4833 if (Offset < 0) 4834 return SDValue(); 4835 if ((Offset % RequiredAlign) & 3) 4836 return SDValue(); 4837 int64_t StartOffset = Offset & ~(RequiredAlign-1); 4838 if (StartOffset) 4839 Ptr = DAG.getNode(ISD::ADD, Ptr.getDebugLoc(), Ptr.getValueType(), 4840 Ptr,DAG.getConstant(StartOffset, Ptr.getValueType())); 4841 4842 int EltNo = (Offset - StartOffset) >> 2; 4843 unsigned NumElems = VT.getVectorNumElements(); 4844 4845 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems); 4846 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr, 4847 LD->getPointerInfo().getWithOffset(StartOffset), 4848 false, false, false, 0); 4849 4850 SmallVector<int, 8> Mask; 4851 for (unsigned i = 0; i != NumElems; ++i) 4852 Mask.push_back(EltNo); 4853 4854 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]); 4855 } 4856 4857 return SDValue(); 4858} 4859 4860/// EltsFromConsecutiveLoads - Given the initializing elements 'Elts' of a 4861/// vector of type 'VT', see if the elements can be replaced by a single large 4862/// load which has the same value as a build_vector whose operands are 'elts'. 4863/// 4864/// Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a 4865/// 4866/// FIXME: we'd also like to handle the case where the last elements are zero 4867/// rather than undef via VZEXT_LOAD, but we do not detect that case today. 4868/// There's even a handy isZeroNode for that purpose. 4869static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts, 4870 DebugLoc &DL, SelectionDAG &DAG) { 4871 EVT EltVT = VT.getVectorElementType(); 4872 unsigned NumElems = Elts.size(); 4873 4874 LoadSDNode *LDBase = NULL; 4875 unsigned LastLoadedElt = -1U; 4876 4877 // For each element in the initializer, see if we've found a load or an undef. 4878 // If we don't find an initial load element, or later load elements are 4879 // non-consecutive, bail out. 4880 for (unsigned i = 0; i < NumElems; ++i) { 4881 SDValue Elt = Elts[i]; 4882 4883 if (!Elt.getNode() || 4884 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) 4885 return SDValue(); 4886 if (!LDBase) { 4887 if (Elt.getNode()->getOpcode() == ISD::UNDEF) 4888 return SDValue(); 4889 LDBase = cast<LoadSDNode>(Elt.getNode()); 4890 LastLoadedElt = i; 4891 continue; 4892 } 4893 if (Elt.getOpcode() == ISD::UNDEF) 4894 continue; 4895 4896 LoadSDNode *LD = cast<LoadSDNode>(Elt); 4897 if (!DAG.isConsecutiveLoad(LD, LDBase, EltVT.getSizeInBits()/8, i)) 4898 return SDValue(); 4899 LastLoadedElt = i; 4900 } 4901 4902 // If we have found an entire vector of loads and undefs, then return a large 4903 // load of the entire vector width starting at the base pointer. If we found 4904 // consecutive loads for the low half, generate a vzext_load node. 4905 if (LastLoadedElt == NumElems - 1) { 4906 if (DAG.InferPtrAlignment(LDBase->getBasePtr()) >= 16) 4907 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), 4908 LDBase->getPointerInfo(), 4909 LDBase->isVolatile(), LDBase->isNonTemporal(), 4910 LDBase->isInvariant(), 0); 4911 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), 4912 LDBase->getPointerInfo(), 4913 LDBase->isVolatile(), LDBase->isNonTemporal(), 4914 LDBase->isInvariant(), LDBase->getAlignment()); 4915 } 4916 if (NumElems == 4 && LastLoadedElt == 1 && 4917 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { 4918 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); 4919 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() }; 4920 SDValue ResNode = 4921 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, 2, MVT::i64, 4922 LDBase->getPointerInfo(), 4923 LDBase->getAlignment(), 4924 false/*isVolatile*/, true/*ReadMem*/, 4925 false/*WriteMem*/); 4926 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); 4927 } 4928 return SDValue(); 4929} 4930 4931/// LowerVectorBroadcast - Attempt to use the vbroadcast instruction 4932/// to generate a splat value for the following cases: 4933/// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant. 4934/// 2. A splat shuffle which uses a scalar_to_vector node which comes from 4935/// a scalar load, or a constant. 4936/// The VBROADCAST node is returned when a pattern is found, 4937/// or SDValue() otherwise. 4938SDValue 4939X86TargetLowering::LowerVectorBroadcast(SDValue &Op, SelectionDAG &DAG) const { 4940 if (!Subtarget->hasAVX()) 4941 return SDValue(); 4942 4943 EVT VT = Op.getValueType(); 4944 DebugLoc dl = Op.getDebugLoc(); 4945 4946 assert((VT.is128BitVector() || VT.is256BitVector()) && 4947 "Unsupported vector type for broadcast."); 4948 4949 SDValue Ld; 4950 bool ConstSplatVal; 4951 4952 switch (Op.getOpcode()) { 4953 default: 4954 // Unknown pattern found. 4955 return SDValue(); 4956 4957 case ISD::BUILD_VECTOR: { 4958 // The BUILD_VECTOR node must be a splat. 4959 if (!isSplatVector(Op.getNode())) 4960 return SDValue(); 4961 4962 Ld = Op.getOperand(0); 4963 ConstSplatVal = (Ld.getOpcode() == ISD::Constant || 4964 Ld.getOpcode() == ISD::ConstantFP); 4965 4966 // The suspected load node has several users. Make sure that all 4967 // of its users are from the BUILD_VECTOR node. 4968 // Constants may have multiple users. 4969 if (!ConstSplatVal && !Ld->hasNUsesOfValue(VT.getVectorNumElements(), 0)) 4970 return SDValue(); 4971 break; 4972 } 4973 4974 case ISD::VECTOR_SHUFFLE: { 4975 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 4976 4977 // Shuffles must have a splat mask where the first element is 4978 // broadcasted. 4979 if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0) 4980 return SDValue(); 4981 4982 SDValue Sc = Op.getOperand(0); 4983 if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR) 4984 return SDValue(); 4985 4986 Ld = Sc.getOperand(0); 4987 ConstSplatVal = (Ld.getOpcode() == ISD::Constant || 4988 Ld.getOpcode() == ISD::ConstantFP); 4989 4990 // The scalar_to_vector node and the suspected 4991 // load node must have exactly one user. 4992 // Constants may have multiple users. 4993 if (!ConstSplatVal && (!Sc.hasOneUse() || !Ld.hasOneUse())) 4994 return SDValue(); 4995 break; 4996 } 4997 } 4998 4999 bool Is256 = VT.getSizeInBits() == 256; 5000 5001 // Handle the broadcasting a single constant scalar from the constant pool 5002 // into a vector. On Sandybridge it is still better to load a constant vector 5003 // from the constant pool and not to broadcast it from a scalar. 5004 if (ConstSplatVal && Subtarget->hasAVX2()) { 5005 EVT CVT = Ld.getValueType(); 5006 assert(!CVT.isVector() && "Must not broadcast a vector type"); 5007 unsigned ScalarSize = CVT.getSizeInBits(); 5008 5009 if (ScalarSize == 32 || (Is256 && ScalarSize == 64)) { 5010 const Constant *C = 0; 5011 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld)) 5012 C = CI->getConstantIntValue(); 5013 else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld)) 5014 C = CF->getConstantFPValue(); 5015 5016 assert(C && "Invalid constant type"); 5017 5018 SDValue CP = DAG.getConstantPool(C, getPointerTy()); 5019 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment(); 5020 Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP, 5021 MachinePointerInfo::getConstantPool(), 5022 false, false, false, Alignment); 5023 5024 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); 5025 } 5026 } 5027 5028 // The scalar source must be a normal load. 5029 if (!ISD::isNormalLoad(Ld.getNode())) 5030 return SDValue(); 5031 5032 // Reject loads that have uses of the chain result 5033 if (Ld->hasAnyUseOfValue(1)) 5034 return SDValue(); 5035 5036 unsigned ScalarSize = Ld.getValueType().getSizeInBits(); 5037 5038 if (ScalarSize == 32 || (Is256 && ScalarSize == 64)) 5039 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); 5040 5041 // The integer check is needed for the 64-bit into 128-bit so it doesn't match 5042 // double since there is no vbroadcastsd xmm 5043 if (Subtarget->hasAVX2() && Ld.getValueType().isInteger()) { 5044 if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64) 5045 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); 5046 } 5047 5048 // Unsupported broadcast. 5049 return SDValue(); 5050} 5051 5052SDValue 5053X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { 5054 DebugLoc dl = Op.getDebugLoc(); 5055 5056 EVT VT = Op.getValueType(); 5057 EVT ExtVT = VT.getVectorElementType(); 5058 unsigned NumElems = Op.getNumOperands(); 5059 5060 // Vectors containing all zeros can be matched by pxor and xorps later 5061 if (ISD::isBuildVectorAllZeros(Op.getNode())) { 5062 // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd 5063 // and 2) ensure that i64 scalars are eliminated on x86-32 hosts. 5064 if (VT == MVT::v4i32 || VT == MVT::v8i32) 5065 return Op; 5066 5067 return getZeroVector(VT, Subtarget, DAG, dl); 5068 } 5069 5070 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width 5071 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use 5072 // vpcmpeqd on 256-bit vectors. 5073 if (ISD::isBuildVectorAllOnes(Op.getNode())) { 5074 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasAVX2())) 5075 return Op; 5076 5077 return getOnesVector(VT, Subtarget->hasAVX2(), DAG, dl); 5078 } 5079 5080 SDValue Broadcast = LowerVectorBroadcast(Op, DAG); 5081 if (Broadcast.getNode()) 5082 return Broadcast; 5083 5084 unsigned EVTBits = ExtVT.getSizeInBits(); 5085 5086 unsigned NumZero = 0; 5087 unsigned NumNonZero = 0; 5088 unsigned NonZeros = 0; 5089 bool IsAllConstants = true; 5090 SmallSet<SDValue, 8> Values; 5091 for (unsigned i = 0; i < NumElems; ++i) { 5092 SDValue Elt = Op.getOperand(i); 5093 if (Elt.getOpcode() == ISD::UNDEF) 5094 continue; 5095 Values.insert(Elt); 5096 if (Elt.getOpcode() != ISD::Constant && 5097 Elt.getOpcode() != ISD::ConstantFP) 5098 IsAllConstants = false; 5099 if (X86::isZeroNode(Elt)) 5100 NumZero++; 5101 else { 5102 NonZeros |= (1 << i); 5103 NumNonZero++; 5104 } 5105 } 5106 5107 // All undef vector. Return an UNDEF. All zero vectors were handled above. 5108 if (NumNonZero == 0) 5109 return DAG.getUNDEF(VT); 5110 5111 // Special case for single non-zero, non-undef, element. 5112 if (NumNonZero == 1) { 5113 unsigned Idx = CountTrailingZeros_32(NonZeros); 5114 SDValue Item = Op.getOperand(Idx); 5115 5116 // If this is an insertion of an i64 value on x86-32, and if the top bits of 5117 // the value are obviously zero, truncate the value to i32 and do the 5118 // insertion that way. Only do this if the value is non-constant or if the 5119 // value is a constant being inserted into element 0. It is cheaper to do 5120 // a constant pool load than it is to do a movd + shuffle. 5121 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && 5122 (!IsAllConstants || Idx == 0)) { 5123 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 5124 // Handle SSE only. 5125 assert(VT == MVT::v2i64 && "Expected an SSE value type!"); 5126 EVT VecVT = MVT::v4i32; 5127 unsigned VecElts = 4; 5128 5129 // Truncate the value (which may itself be a constant) to i32, and 5130 // convert it to a vector with movd (S2V+shuffle to zero extend). 5131 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); 5132 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); 5133 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); 5134 5135 // Now we have our 32-bit value zero extended in the low element of 5136 // a vector. If Idx != 0, swizzle it into place. 5137 if (Idx != 0) { 5138 SmallVector<int, 4> Mask; 5139 Mask.push_back(Idx); 5140 for (unsigned i = 1; i != VecElts; ++i) 5141 Mask.push_back(i); 5142 Item = DAG.getVectorShuffle(VecVT, dl, Item, DAG.getUNDEF(VecVT), 5143 &Mask[0]); 5144 } 5145 return DAG.getNode(ISD::BITCAST, dl, VT, Item); 5146 } 5147 } 5148 5149 // If we have a constant or non-constant insertion into the low element of 5150 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 5151 // the rest of the elements. This will be matched as movd/movq/movss/movsd 5152 // depending on what the source datatype is. 5153 if (Idx == 0) { 5154 if (NumZero == 0) 5155 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 5156 5157 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || 5158 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { 5159 if (VT.getSizeInBits() == 256) { 5160 SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl); 5161 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec, 5162 Item, DAG.getIntPtrConstant(0)); 5163 } 5164 assert(VT.getSizeInBits() == 128 && "Expected an SSE value type!"); 5165 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 5166 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 5167 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); 5168 } 5169 5170 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { 5171 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); 5172 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); 5173 if (VT.getSizeInBits() == 256) { 5174 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); 5175 Item = Insert128BitVector(ZeroVec, Item, 0, DAG, dl); 5176 } else { 5177 assert(VT.getSizeInBits() == 128 && "Expected an SSE value type!"); 5178 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); 5179 } 5180 return DAG.getNode(ISD::BITCAST, dl, VT, Item); 5181 } 5182 } 5183 5184 // Is it a vector logical left shift? 5185 if (NumElems == 2 && Idx == 1 && 5186 X86::isZeroNode(Op.getOperand(0)) && 5187 !X86::isZeroNode(Op.getOperand(1))) { 5188 unsigned NumBits = VT.getSizeInBits(); 5189 return getVShift(true, VT, 5190 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 5191 VT, Op.getOperand(1)), 5192 NumBits/2, DAG, *this, dl); 5193 } 5194 5195 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 5196 return SDValue(); 5197 5198 // Otherwise, if this is a vector with i32 or f32 elements, and the element 5199 // is a non-constant being inserted into an element other than the low one, 5200 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 5201 // movd/movss) to move this into the low element, then shuffle it into 5202 // place. 5203 if (EVTBits == 32) { 5204 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 5205 5206 // Turn it into a shuffle of zero and zero-extended scalar to vector. 5207 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, Subtarget, DAG); 5208 SmallVector<int, 8> MaskVec; 5209 for (unsigned i = 0; i != NumElems; ++i) 5210 MaskVec.push_back(i == Idx ? 0 : 1); 5211 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]); 5212 } 5213 } 5214 5215 // Splat is obviously ok. Let legalizer expand it to a shuffle. 5216 if (Values.size() == 1) { 5217 if (EVTBits == 32) { 5218 // Instead of a shuffle like this: 5219 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> 5220 // Check if it's possible to issue this instead. 5221 // shuffle (vload ptr)), undef, <1, 1, 1, 1> 5222 unsigned Idx = CountTrailingZeros_32(NonZeros); 5223 SDValue Item = Op.getOperand(Idx); 5224 if (Op.getNode()->isOnlyUserOf(Item.getNode())) 5225 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); 5226 } 5227 return SDValue(); 5228 } 5229 5230 // A vector full of immediates; various special cases are already 5231 // handled, so this is best done with a single constant-pool load. 5232 if (IsAllConstants) 5233 return SDValue(); 5234 5235 // For AVX-length vectors, build the individual 128-bit pieces and use 5236 // shuffles to put them in place. 5237 if (VT.getSizeInBits() == 256) { 5238 SmallVector<SDValue, 32> V; 5239 for (unsigned i = 0; i != NumElems; ++i) 5240 V.push_back(Op.getOperand(i)); 5241 5242 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2); 5243 5244 // Build both the lower and upper subvector. 5245 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[0], NumElems/2); 5246 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[NumElems / 2], 5247 NumElems/2); 5248 5249 // Recreate the wider vector with the lower and upper part. 5250 return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl); 5251 } 5252 5253 // Let legalizer expand 2-wide build_vectors. 5254 if (EVTBits == 64) { 5255 if (NumNonZero == 1) { 5256 // One half is zero or undef. 5257 unsigned Idx = CountTrailingZeros_32(NonZeros); 5258 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, 5259 Op.getOperand(Idx)); 5260 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); 5261 } 5262 return SDValue(); 5263 } 5264 5265 // If element VT is < 32 bits, convert it to inserts into a zero vector. 5266 if (EVTBits == 8 && NumElems == 16) { 5267 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 5268 Subtarget, *this); 5269 if (V.getNode()) return V; 5270 } 5271 5272 if (EVTBits == 16 && NumElems == 8) { 5273 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 5274 Subtarget, *this); 5275 if (V.getNode()) return V; 5276 } 5277 5278 // If element VT is == 32 bits, turn it into a number of shuffles. 5279 SmallVector<SDValue, 8> V(NumElems); 5280 if (NumElems == 4 && NumZero > 0) { 5281 for (unsigned i = 0; i < 4; ++i) { 5282 bool isZero = !(NonZeros & (1 << i)); 5283 if (isZero) 5284 V[i] = getZeroVector(VT, Subtarget, DAG, dl); 5285 else 5286 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 5287 } 5288 5289 for (unsigned i = 0; i < 2; ++i) { 5290 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 5291 default: break; 5292 case 0: 5293 V[i] = V[i*2]; // Must be a zero vector. 5294 break; 5295 case 1: 5296 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); 5297 break; 5298 case 2: 5299 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); 5300 break; 5301 case 3: 5302 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); 5303 break; 5304 } 5305 } 5306 5307 bool Reverse1 = (NonZeros & 0x3) == 2; 5308 bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2; 5309 int MaskVec[] = { 5310 Reverse1 ? 1 : 0, 5311 Reverse1 ? 0 : 1, 5312 static_cast<int>(Reverse2 ? NumElems+1 : NumElems), 5313 static_cast<int>(Reverse2 ? NumElems : NumElems+1) 5314 }; 5315 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); 5316 } 5317 5318 if (Values.size() > 1 && VT.getSizeInBits() == 128) { 5319 // Check for a build vector of consecutive loads. 5320 for (unsigned i = 0; i < NumElems; ++i) 5321 V[i] = Op.getOperand(i); 5322 5323 // Check for elements which are consecutive loads. 5324 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG); 5325 if (LD.getNode()) 5326 return LD; 5327 5328 // For SSE 4.1, use insertps to put the high elements into the low element. 5329 if (getSubtarget()->hasSSE41()) { 5330 SDValue Result; 5331 if (Op.getOperand(0).getOpcode() != ISD::UNDEF) 5332 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); 5333 else 5334 Result = DAG.getUNDEF(VT); 5335 5336 for (unsigned i = 1; i < NumElems; ++i) { 5337 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue; 5338 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, 5339 Op.getOperand(i), DAG.getIntPtrConstant(i)); 5340 } 5341 return Result; 5342 } 5343 5344 // Otherwise, expand into a number of unpckl*, start by extending each of 5345 // our (non-undef) elements to the full vector width with the element in the 5346 // bottom slot of the vector (which generates no code for SSE). 5347 for (unsigned i = 0; i < NumElems; ++i) { 5348 if (Op.getOperand(i).getOpcode() != ISD::UNDEF) 5349 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 5350 else 5351 V[i] = DAG.getUNDEF(VT); 5352 } 5353 5354 // Next, we iteratively mix elements, e.g. for v4f32: 5355 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 5356 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 5357 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 5358 unsigned EltStride = NumElems >> 1; 5359 while (EltStride != 0) { 5360 for (unsigned i = 0; i < EltStride; ++i) { 5361 // If V[i+EltStride] is undef and this is the first round of mixing, 5362 // then it is safe to just drop this shuffle: V[i] is already in the 5363 // right place, the one element (since it's the first round) being 5364 // inserted as undef can be dropped. This isn't safe for successive 5365 // rounds because they will permute elements within both vectors. 5366 if (V[i+EltStride].getOpcode() == ISD::UNDEF && 5367 EltStride == NumElems/2) 5368 continue; 5369 5370 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]); 5371 } 5372 EltStride >>= 1; 5373 } 5374 return V[0]; 5375 } 5376 return SDValue(); 5377} 5378 5379// LowerMMXCONCAT_VECTORS - We support concatenate two MMX registers and place 5380// them in a MMX register. This is better than doing a stack convert. 5381static SDValue LowerMMXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { 5382 DebugLoc dl = Op.getDebugLoc(); 5383 EVT ResVT = Op.getValueType(); 5384 5385 assert(ResVT == MVT::v2i64 || ResVT == MVT::v4i32 || 5386 ResVT == MVT::v8i16 || ResVT == MVT::v16i8); 5387 int Mask[2]; 5388 SDValue InVec = DAG.getNode(ISD::BITCAST,dl, MVT::v1i64, Op.getOperand(0)); 5389 SDValue VecOp = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); 5390 InVec = Op.getOperand(1); 5391 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) { 5392 unsigned NumElts = ResVT.getVectorNumElements(); 5393 VecOp = DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); 5394 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ResVT, VecOp, 5395 InVec.getOperand(0), DAG.getIntPtrConstant(NumElts/2+1)); 5396 } else { 5397 InVec = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, InVec); 5398 SDValue VecOp2 = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); 5399 Mask[0] = 0; Mask[1] = 2; 5400 VecOp = DAG.getVectorShuffle(MVT::v2i64, dl, VecOp, VecOp2, Mask); 5401 } 5402 return DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); 5403} 5404 5405// LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction 5406// to create 256-bit vectors from two other 128-bit ones. 5407static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { 5408 DebugLoc dl = Op.getDebugLoc(); 5409 EVT ResVT = Op.getValueType(); 5410 5411 assert(ResVT.getSizeInBits() == 256 && "Value type must be 256-bit wide"); 5412 5413 SDValue V1 = Op.getOperand(0); 5414 SDValue V2 = Op.getOperand(1); 5415 unsigned NumElems = ResVT.getVectorNumElements(); 5416 5417 return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl); 5418} 5419 5420SDValue 5421X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const { 5422 EVT ResVT = Op.getValueType(); 5423 5424 assert(Op.getNumOperands() == 2); 5425 assert((ResVT.getSizeInBits() == 128 || ResVT.getSizeInBits() == 256) && 5426 "Unsupported CONCAT_VECTORS for value type"); 5427 5428 // We support concatenate two MMX registers and place them in a MMX register. 5429 // This is better than doing a stack convert. 5430 if (ResVT.is128BitVector()) 5431 return LowerMMXCONCAT_VECTORS(Op, DAG); 5432 5433 // 256-bit AVX can use the vinsertf128 instruction to create 256-bit vectors 5434 // from two other 128-bit ones. 5435 return LowerAVXCONCAT_VECTORS(Op, DAG); 5436} 5437 5438// Try to lower a shuffle node into a simple blend instruction. 5439static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp, 5440 const X86Subtarget *Subtarget, 5441 SelectionDAG &DAG) { 5442 SDValue V1 = SVOp->getOperand(0); 5443 SDValue V2 = SVOp->getOperand(1); 5444 DebugLoc dl = SVOp->getDebugLoc(); 5445 MVT VT = SVOp->getValueType(0).getSimpleVT(); 5446 unsigned NumElems = VT.getVectorNumElements(); 5447 5448 if (!Subtarget->hasSSE41()) 5449 return SDValue(); 5450 5451 unsigned ISDNo = 0; 5452 MVT OpTy; 5453 5454 switch (VT.SimpleTy) { 5455 default: return SDValue(); 5456 case MVT::v8i16: 5457 ISDNo = X86ISD::BLENDPW; 5458 OpTy = MVT::v8i16; 5459 break; 5460 case MVT::v4i32: 5461 case MVT::v4f32: 5462 ISDNo = X86ISD::BLENDPS; 5463 OpTy = MVT::v4f32; 5464 break; 5465 case MVT::v2i64: 5466 case MVT::v2f64: 5467 ISDNo = X86ISD::BLENDPD; 5468 OpTy = MVT::v2f64; 5469 break; 5470 case MVT::v8i32: 5471 case MVT::v8f32: 5472 if (!Subtarget->hasAVX()) 5473 return SDValue(); 5474 ISDNo = X86ISD::BLENDPS; 5475 OpTy = MVT::v8f32; 5476 break; 5477 case MVT::v4i64: 5478 case MVT::v4f64: 5479 if (!Subtarget->hasAVX()) 5480 return SDValue(); 5481 ISDNo = X86ISD::BLENDPD; 5482 OpTy = MVT::v4f64; 5483 break; 5484 } 5485 assert(ISDNo && "Invalid Op Number"); 5486 5487 unsigned MaskVals = 0; 5488 5489 for (unsigned i = 0; i != NumElems; ++i) { 5490 int EltIdx = SVOp->getMaskElt(i); 5491 if (EltIdx == (int)i || EltIdx < 0) 5492 MaskVals |= (1<<i); 5493 else if (EltIdx == (int)(i + NumElems)) 5494 continue; // Bit is set to zero; 5495 else 5496 return SDValue(); 5497 } 5498 5499 V1 = DAG.getNode(ISD::BITCAST, dl, OpTy, V1); 5500 V2 = DAG.getNode(ISD::BITCAST, dl, OpTy, V2); 5501 SDValue Ret = DAG.getNode(ISDNo, dl, OpTy, V1, V2, 5502 DAG.getConstant(MaskVals, MVT::i32)); 5503 return DAG.getNode(ISD::BITCAST, dl, VT, Ret); 5504} 5505 5506// v8i16 shuffles - Prefer shuffles in the following order: 5507// 1. [all] pshuflw, pshufhw, optional move 5508// 2. [ssse3] 1 x pshufb 5509// 3. [ssse3] 2 x pshufb + 1 x por 5510// 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw) 5511SDValue 5512X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op, 5513 SelectionDAG &DAG) const { 5514 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 5515 SDValue V1 = SVOp->getOperand(0); 5516 SDValue V2 = SVOp->getOperand(1); 5517 DebugLoc dl = SVOp->getDebugLoc(); 5518 SmallVector<int, 8> MaskVals; 5519 5520 // Determine if more than 1 of the words in each of the low and high quadwords 5521 // of the result come from the same quadword of one of the two inputs. Undef 5522 // mask values count as coming from any quadword, for better codegen. 5523 unsigned LoQuad[] = { 0, 0, 0, 0 }; 5524 unsigned HiQuad[] = { 0, 0, 0, 0 }; 5525 std::bitset<4> InputQuads; 5526 for (unsigned i = 0; i < 8; ++i) { 5527 unsigned *Quad = i < 4 ? LoQuad : HiQuad; 5528 int EltIdx = SVOp->getMaskElt(i); 5529 MaskVals.push_back(EltIdx); 5530 if (EltIdx < 0) { 5531 ++Quad[0]; 5532 ++Quad[1]; 5533 ++Quad[2]; 5534 ++Quad[3]; 5535 continue; 5536 } 5537 ++Quad[EltIdx / 4]; 5538 InputQuads.set(EltIdx / 4); 5539 } 5540 5541 int BestLoQuad = -1; 5542 unsigned MaxQuad = 1; 5543 for (unsigned i = 0; i < 4; ++i) { 5544 if (LoQuad[i] > MaxQuad) { 5545 BestLoQuad = i; 5546 MaxQuad = LoQuad[i]; 5547 } 5548 } 5549 5550 int BestHiQuad = -1; 5551 MaxQuad = 1; 5552 for (unsigned i = 0; i < 4; ++i) { 5553 if (HiQuad[i] > MaxQuad) { 5554 BestHiQuad = i; 5555 MaxQuad = HiQuad[i]; 5556 } 5557 } 5558 5559 // For SSSE3, If all 8 words of the result come from only 1 quadword of each 5560 // of the two input vectors, shuffle them into one input vector so only a 5561 // single pshufb instruction is necessary. If There are more than 2 input 5562 // quads, disable the next transformation since it does not help SSSE3. 5563 bool V1Used = InputQuads[0] || InputQuads[1]; 5564 bool V2Used = InputQuads[2] || InputQuads[3]; 5565 if (Subtarget->hasSSSE3()) { 5566 if (InputQuads.count() == 2 && V1Used && V2Used) { 5567 BestLoQuad = InputQuads[0] ? 0 : 1; 5568 BestHiQuad = InputQuads[2] ? 2 : 3; 5569 } 5570 if (InputQuads.count() > 2) { 5571 BestLoQuad = -1; 5572 BestHiQuad = -1; 5573 } 5574 } 5575 5576 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update 5577 // the shuffle mask. If a quad is scored as -1, that means that it contains 5578 // words from all 4 input quadwords. 5579 SDValue NewV; 5580 if (BestLoQuad >= 0 || BestHiQuad >= 0) { 5581 int MaskV[] = { 5582 BestLoQuad < 0 ? 0 : BestLoQuad, 5583 BestHiQuad < 0 ? 1 : BestHiQuad 5584 }; 5585 NewV = DAG.getVectorShuffle(MVT::v2i64, dl, 5586 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1), 5587 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]); 5588 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV); 5589 5590 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the 5591 // source words for the shuffle, to aid later transformations. 5592 bool AllWordsInNewV = true; 5593 bool InOrder[2] = { true, true }; 5594 for (unsigned i = 0; i != 8; ++i) { 5595 int idx = MaskVals[i]; 5596 if (idx != (int)i) 5597 InOrder[i/4] = false; 5598 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad) 5599 continue; 5600 AllWordsInNewV = false; 5601 break; 5602 } 5603 5604 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV; 5605 if (AllWordsInNewV) { 5606 for (int i = 0; i != 8; ++i) { 5607 int idx = MaskVals[i]; 5608 if (idx < 0) 5609 continue; 5610 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4; 5611 if ((idx != i) && idx < 4) 5612 pshufhw = false; 5613 if ((idx != i) && idx > 3) 5614 pshuflw = false; 5615 } 5616 V1 = NewV; 5617 V2Used = false; 5618 BestLoQuad = 0; 5619 BestHiQuad = 1; 5620 } 5621 5622 // If we've eliminated the use of V2, and the new mask is a pshuflw or 5623 // pshufhw, that's as cheap as it gets. Return the new shuffle. 5624 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) { 5625 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW; 5626 unsigned TargetMask = 0; 5627 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, 5628 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); 5629 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode()); 5630 TargetMask = pshufhw ? getShufflePSHUFHWImmediate(SVOp): 5631 getShufflePSHUFLWImmediate(SVOp); 5632 V1 = NewV.getOperand(0); 5633 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG); 5634 } 5635 } 5636 5637 // If we have SSSE3, and all words of the result are from 1 input vector, 5638 // case 2 is generated, otherwise case 3 is generated. If no SSSE3 5639 // is present, fall back to case 4. 5640 if (Subtarget->hasSSSE3()) { 5641 SmallVector<SDValue,16> pshufbMask; 5642 5643 // If we have elements from both input vectors, set the high bit of the 5644 // shuffle mask element to zero out elements that come from V2 in the V1 5645 // mask, and elements that come from V1 in the V2 mask, so that the two 5646 // results can be OR'd together. 5647 bool TwoInputs = V1Used && V2Used; 5648 for (unsigned i = 0; i != 8; ++i) { 5649 int EltIdx = MaskVals[i] * 2; 5650 if (TwoInputs && (EltIdx >= 16)) { 5651 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5652 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5653 continue; 5654 } 5655 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 5656 pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8)); 5657 } 5658 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V1); 5659 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 5660 DAG.getNode(ISD::BUILD_VECTOR, dl, 5661 MVT::v16i8, &pshufbMask[0], 16)); 5662 if (!TwoInputs) 5663 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5664 5665 // Calculate the shuffle mask for the second input, shuffle it, and 5666 // OR it with the first shuffled input. 5667 pshufbMask.clear(); 5668 for (unsigned i = 0; i != 8; ++i) { 5669 int EltIdx = MaskVals[i] * 2; 5670 if (EltIdx < 16) { 5671 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5672 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5673 continue; 5674 } 5675 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 5676 pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8)); 5677 } 5678 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V2); 5679 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 5680 DAG.getNode(ISD::BUILD_VECTOR, dl, 5681 MVT::v16i8, &pshufbMask[0], 16)); 5682 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 5683 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5684 } 5685 5686 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order, 5687 // and update MaskVals with new element order. 5688 std::bitset<8> InOrder; 5689 if (BestLoQuad >= 0) { 5690 int MaskV[] = { -1, -1, -1, -1, 4, 5, 6, 7 }; 5691 for (int i = 0; i != 4; ++i) { 5692 int idx = MaskVals[i]; 5693 if (idx < 0) { 5694 InOrder.set(i); 5695 } else if ((idx / 4) == BestLoQuad) { 5696 MaskV[i] = idx & 3; 5697 InOrder.set(i); 5698 } 5699 } 5700 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 5701 &MaskV[0]); 5702 5703 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3()) { 5704 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode()); 5705 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16, 5706 NewV.getOperand(0), 5707 getShufflePSHUFLWImmediate(SVOp), DAG); 5708 } 5709 } 5710 5711 // If BestHi >= 0, generate a pshufhw to put the high elements in order, 5712 // and update MaskVals with the new element order. 5713 if (BestHiQuad >= 0) { 5714 int MaskV[] = { 0, 1, 2, 3, -1, -1, -1, -1 }; 5715 for (unsigned i = 4; i != 8; ++i) { 5716 int idx = MaskVals[i]; 5717 if (idx < 0) { 5718 InOrder.set(i); 5719 } else if ((idx / 4) == BestHiQuad) { 5720 MaskV[i] = (idx & 3) + 4; 5721 InOrder.set(i); 5722 } 5723 } 5724 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 5725 &MaskV[0]); 5726 5727 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSSE3()) { 5728 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode()); 5729 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16, 5730 NewV.getOperand(0), 5731 getShufflePSHUFHWImmediate(SVOp), DAG); 5732 } 5733 } 5734 5735 // In case BestHi & BestLo were both -1, which means each quadword has a word 5736 // from each of the four input quadwords, calculate the InOrder bitvector now 5737 // before falling through to the insert/extract cleanup. 5738 if (BestLoQuad == -1 && BestHiQuad == -1) { 5739 NewV = V1; 5740 for (int i = 0; i != 8; ++i) 5741 if (MaskVals[i] < 0 || MaskVals[i] == i) 5742 InOrder.set(i); 5743 } 5744 5745 // The other elements are put in the right place using pextrw and pinsrw. 5746 for (unsigned i = 0; i != 8; ++i) { 5747 if (InOrder[i]) 5748 continue; 5749 int EltIdx = MaskVals[i]; 5750 if (EltIdx < 0) 5751 continue; 5752 SDValue ExtOp = (EltIdx < 8) ? 5753 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, 5754 DAG.getIntPtrConstant(EltIdx)) : 5755 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, 5756 DAG.getIntPtrConstant(EltIdx - 8)); 5757 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, 5758 DAG.getIntPtrConstant(i)); 5759 } 5760 return NewV; 5761} 5762 5763// v16i8 shuffles - Prefer shuffles in the following order: 5764// 1. [ssse3] 1 x pshufb 5765// 2. [ssse3] 2 x pshufb + 1 x por 5766// 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw 5767static 5768SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp, 5769 SelectionDAG &DAG, 5770 const X86TargetLowering &TLI) { 5771 SDValue V1 = SVOp->getOperand(0); 5772 SDValue V2 = SVOp->getOperand(1); 5773 DebugLoc dl = SVOp->getDebugLoc(); 5774 ArrayRef<int> MaskVals = SVOp->getMask(); 5775 5776 // If we have SSSE3, case 1 is generated when all result bytes come from 5777 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is 5778 // present, fall back to case 3. 5779 // FIXME: kill V2Only once shuffles are canonizalized by getNode. 5780 bool V1Only = true; 5781 bool V2Only = true; 5782 for (unsigned i = 0; i < 16; ++i) { 5783 int EltIdx = MaskVals[i]; 5784 if (EltIdx < 0) 5785 continue; 5786 if (EltIdx < 16) 5787 V2Only = false; 5788 else 5789 V1Only = false; 5790 } 5791 5792 // If SSSE3, use 1 pshufb instruction per vector with elements in the result. 5793 if (TLI.getSubtarget()->hasSSSE3()) { 5794 SmallVector<SDValue,16> pshufbMask; 5795 5796 // If all result elements are from one input vector, then only translate 5797 // undef mask values to 0x80 (zero out result) in the pshufb mask. 5798 // 5799 // Otherwise, we have elements from both input vectors, and must zero out 5800 // elements that come from V2 in the first mask, and V1 in the second mask 5801 // so that we can OR them together. 5802 bool TwoInputs = !(V1Only || V2Only); 5803 for (unsigned i = 0; i != 16; ++i) { 5804 int EltIdx = MaskVals[i]; 5805 if (EltIdx < 0 || (TwoInputs && EltIdx >= 16)) { 5806 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5807 continue; 5808 } 5809 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 5810 } 5811 // If all the elements are from V2, assign it to V1 and return after 5812 // building the first pshufb. 5813 if (V2Only) 5814 V1 = V2; 5815 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 5816 DAG.getNode(ISD::BUILD_VECTOR, dl, 5817 MVT::v16i8, &pshufbMask[0], 16)); 5818 if (!TwoInputs) 5819 return V1; 5820 5821 // Calculate the shuffle mask for the second input, shuffle it, and 5822 // OR it with the first shuffled input. 5823 pshufbMask.clear(); 5824 for (unsigned i = 0; i != 16; ++i) { 5825 int EltIdx = MaskVals[i]; 5826 if (EltIdx < 16) { 5827 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 5828 continue; 5829 } 5830 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 5831 } 5832 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 5833 DAG.getNode(ISD::BUILD_VECTOR, dl, 5834 MVT::v16i8, &pshufbMask[0], 16)); 5835 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 5836 } 5837 5838 // No SSSE3 - Calculate in place words and then fix all out of place words 5839 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from 5840 // the 16 different words that comprise the two doublequadword input vectors. 5841 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 5842 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); 5843 SDValue NewV = V2Only ? V2 : V1; 5844 for (int i = 0; i != 8; ++i) { 5845 int Elt0 = MaskVals[i*2]; 5846 int Elt1 = MaskVals[i*2+1]; 5847 5848 // This word of the result is all undef, skip it. 5849 if (Elt0 < 0 && Elt1 < 0) 5850 continue; 5851 5852 // This word of the result is already in the correct place, skip it. 5853 if (V1Only && (Elt0 == i*2) && (Elt1 == i*2+1)) 5854 continue; 5855 if (V2Only && (Elt0 == i*2+16) && (Elt1 == i*2+17)) 5856 continue; 5857 5858 SDValue Elt0Src = Elt0 < 16 ? V1 : V2; 5859 SDValue Elt1Src = Elt1 < 16 ? V1 : V2; 5860 SDValue InsElt; 5861 5862 // If Elt0 and Elt1 are defined, are consecutive, and can be load 5863 // using a single extract together, load it and store it. 5864 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) { 5865 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 5866 DAG.getIntPtrConstant(Elt1 / 2)); 5867 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 5868 DAG.getIntPtrConstant(i)); 5869 continue; 5870 } 5871 5872 // If Elt1 is defined, extract it from the appropriate source. If the 5873 // source byte is not also odd, shift the extracted word left 8 bits 5874 // otherwise clear the bottom 8 bits if we need to do an or. 5875 if (Elt1 >= 0) { 5876 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 5877 DAG.getIntPtrConstant(Elt1 / 2)); 5878 if ((Elt1 & 1) == 0) 5879 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, 5880 DAG.getConstant(8, 5881 TLI.getShiftAmountTy(InsElt.getValueType()))); 5882 else if (Elt0 >= 0) 5883 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, 5884 DAG.getConstant(0xFF00, MVT::i16)); 5885 } 5886 // If Elt0 is defined, extract it from the appropriate source. If the 5887 // source byte is not also even, shift the extracted word right 8 bits. If 5888 // Elt1 was also defined, OR the extracted values together before 5889 // inserting them in the result. 5890 if (Elt0 >= 0) { 5891 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, 5892 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2)); 5893 if ((Elt0 & 1) != 0) 5894 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, 5895 DAG.getConstant(8, 5896 TLI.getShiftAmountTy(InsElt0.getValueType()))); 5897 else if (Elt1 >= 0) 5898 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, 5899 DAG.getConstant(0x00FF, MVT::i16)); 5900 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) 5901 : InsElt0; 5902 } 5903 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 5904 DAG.getIntPtrConstant(i)); 5905 } 5906 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV); 5907} 5908 5909/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 5910/// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be 5911/// done when every pair / quad of shuffle mask elements point to elements in 5912/// the right sequence. e.g. 5913/// vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15> 5914static 5915SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp, 5916 SelectionDAG &DAG, DebugLoc dl) { 5917 MVT VT = SVOp->getValueType(0).getSimpleVT(); 5918 unsigned NumElems = VT.getVectorNumElements(); 5919 MVT NewVT; 5920 unsigned Scale; 5921 switch (VT.SimpleTy) { 5922 default: llvm_unreachable("Unexpected!"); 5923 case MVT::v4f32: NewVT = MVT::v2f64; Scale = 2; break; 5924 case MVT::v4i32: NewVT = MVT::v2i64; Scale = 2; break; 5925 case MVT::v8i16: NewVT = MVT::v4i32; Scale = 2; break; 5926 case MVT::v16i8: NewVT = MVT::v4i32; Scale = 4; break; 5927 case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break; 5928 case MVT::v32i8: NewVT = MVT::v8i32; Scale = 4; break; 5929 } 5930 5931 SmallVector<int, 8> MaskVec; 5932 for (unsigned i = 0; i != NumElems; i += Scale) { 5933 int StartIdx = -1; 5934 for (unsigned j = 0; j != Scale; ++j) { 5935 int EltIdx = SVOp->getMaskElt(i+j); 5936 if (EltIdx < 0) 5937 continue; 5938 if (StartIdx < 0) 5939 StartIdx = (EltIdx / Scale); 5940 if (EltIdx != (int)(StartIdx*Scale + j)) 5941 return SDValue(); 5942 } 5943 MaskVec.push_back(StartIdx); 5944 } 5945 5946 SDValue V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(0)); 5947 SDValue V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(1)); 5948 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]); 5949} 5950 5951/// getVZextMovL - Return a zero-extending vector move low node. 5952/// 5953static SDValue getVZextMovL(EVT VT, EVT OpVT, 5954 SDValue SrcOp, SelectionDAG &DAG, 5955 const X86Subtarget *Subtarget, DebugLoc dl) { 5956 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 5957 LoadSDNode *LD = NULL; 5958 if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) 5959 LD = dyn_cast<LoadSDNode>(SrcOp); 5960 if (!LD) { 5961 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 5962 // instead. 5963 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 5964 if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) && 5965 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 5966 SrcOp.getOperand(0).getOpcode() == ISD::BITCAST && 5967 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) { 5968 // PR2108 5969 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 5970 return DAG.getNode(ISD::BITCAST, dl, VT, 5971 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 5972 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 5973 OpVT, 5974 SrcOp.getOperand(0) 5975 .getOperand(0)))); 5976 } 5977 } 5978 } 5979 5980 return DAG.getNode(ISD::BITCAST, dl, VT, 5981 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 5982 DAG.getNode(ISD::BITCAST, dl, 5983 OpVT, SrcOp))); 5984} 5985 5986/// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles 5987/// which could not be matched by any known target speficic shuffle 5988static SDValue 5989LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 5990 EVT VT = SVOp->getValueType(0); 5991 5992 unsigned NumElems = VT.getVectorNumElements(); 5993 unsigned NumLaneElems = NumElems / 2; 5994 5995 DebugLoc dl = SVOp->getDebugLoc(); 5996 MVT EltVT = VT.getVectorElementType().getSimpleVT(); 5997 EVT NVT = MVT::getVectorVT(EltVT, NumLaneElems); 5998 SDValue Shufs[2]; 5999 6000 SmallVector<int, 16> Mask; 6001 for (unsigned l = 0; l < 2; ++l) { 6002 // Build a shuffle mask for the output, discovering on the fly which 6003 // input vectors to use as shuffle operands (recorded in InputUsed). 6004 // If building a suitable shuffle vector proves too hard, then bail 6005 // out with useBuildVector set. 6006 int InputUsed[2] = { -1, -1 }; // Not yet discovered. 6007 unsigned LaneStart = l * NumLaneElems; 6008 for (unsigned i = 0; i != NumLaneElems; ++i) { 6009 // The mask element. This indexes into the input. 6010 int Idx = SVOp->getMaskElt(i+LaneStart); 6011 if (Idx < 0) { 6012 // the mask element does not index into any input vector. 6013 Mask.push_back(-1); 6014 continue; 6015 } 6016 6017 // The input vector this mask element indexes into. 6018 int Input = Idx / NumLaneElems; 6019 6020 // Turn the index into an offset from the start of the input vector. 6021 Idx -= Input * NumLaneElems; 6022 6023 // Find or create a shuffle vector operand to hold this input. 6024 unsigned OpNo; 6025 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 6026 if (InputUsed[OpNo] == Input) 6027 // This input vector is already an operand. 6028 break; 6029 if (InputUsed[OpNo] < 0) { 6030 // Create a new operand for this input vector. 6031 InputUsed[OpNo] = Input; 6032 break; 6033 } 6034 } 6035 6036 if (OpNo >= array_lengthof(InputUsed)) { 6037 // More than two input vectors used! Give up. 6038 return SDValue(); 6039 } 6040 6041 // Add the mask index for the new shuffle vector. 6042 Mask.push_back(Idx + OpNo * NumLaneElems); 6043 } 6044 6045 if (InputUsed[0] < 0) { 6046 // No input vectors were used! The result is undefined. 6047 Shufs[l] = DAG.getUNDEF(NVT); 6048 } else { 6049 SDValue Op0 = Extract128BitVector(SVOp->getOperand(InputUsed[0] / 2), 6050 (InputUsed[0] % 2) * NumLaneElems, 6051 DAG, dl); 6052 // If only one input was used, use an undefined vector for the other. 6053 SDValue Op1 = (InputUsed[1] < 0) ? DAG.getUNDEF(NVT) : 6054 Extract128BitVector(SVOp->getOperand(InputUsed[1] / 2), 6055 (InputUsed[1] % 2) * NumLaneElems, DAG, dl); 6056 // At least one input vector was used. Create a new shuffle vector. 6057 Shufs[l] = DAG.getVectorShuffle(NVT, dl, Op0, Op1, &Mask[0]); 6058 } 6059 6060 Mask.clear(); 6061 } 6062 6063 // Concatenate the result back 6064 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Shufs[0], Shufs[1]); 6065} 6066 6067/// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with 6068/// 4 elements, and match them with several different shuffle types. 6069static SDValue 6070LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 6071 SDValue V1 = SVOp->getOperand(0); 6072 SDValue V2 = SVOp->getOperand(1); 6073 DebugLoc dl = SVOp->getDebugLoc(); 6074 EVT VT = SVOp->getValueType(0); 6075 6076 assert(VT.getSizeInBits() == 128 && "Unsupported vector size"); 6077 6078 std::pair<int, int> Locs[4]; 6079 int Mask1[] = { -1, -1, -1, -1 }; 6080 SmallVector<int, 8> PermMask(SVOp->getMask().begin(), SVOp->getMask().end()); 6081 6082 unsigned NumHi = 0; 6083 unsigned NumLo = 0; 6084 for (unsigned i = 0; i != 4; ++i) { 6085 int Idx = PermMask[i]; 6086 if (Idx < 0) { 6087 Locs[i] = std::make_pair(-1, -1); 6088 } else { 6089 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!"); 6090 if (Idx < 4) { 6091 Locs[i] = std::make_pair(0, NumLo); 6092 Mask1[NumLo] = Idx; 6093 NumLo++; 6094 } else { 6095 Locs[i] = std::make_pair(1, NumHi); 6096 if (2+NumHi < 4) 6097 Mask1[2+NumHi] = Idx; 6098 NumHi++; 6099 } 6100 } 6101 } 6102 6103 if (NumLo <= 2 && NumHi <= 2) { 6104 // If no more than two elements come from either vector. This can be 6105 // implemented with two shuffles. First shuffle gather the elements. 6106 // The second shuffle, which takes the first shuffle as both of its 6107 // vector operands, put the elements into the right order. 6108 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 6109 6110 int Mask2[] = { -1, -1, -1, -1 }; 6111 6112 for (unsigned i = 0; i != 4; ++i) 6113 if (Locs[i].first != -1) { 6114 unsigned Idx = (i < 2) ? 0 : 4; 6115 Idx += Locs[i].first * 2 + Locs[i].second; 6116 Mask2[i] = Idx; 6117 } 6118 6119 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); 6120 } 6121 6122 if (NumLo == 3 || NumHi == 3) { 6123 // Otherwise, we must have three elements from one vector, call it X, and 6124 // one element from the other, call it Y. First, use a shufps to build an 6125 // intermediate vector with the one element from Y and the element from X 6126 // that will be in the same half in the final destination (the indexes don't 6127 // matter). Then, use a shufps to build the final vector, taking the half 6128 // containing the element from Y from the intermediate, and the other half 6129 // from X. 6130 if (NumHi == 3) { 6131 // Normalize it so the 3 elements come from V1. 6132 CommuteVectorShuffleMask(PermMask, 4); 6133 std::swap(V1, V2); 6134 } 6135 6136 // Find the element from V2. 6137 unsigned HiIndex; 6138 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 6139 int Val = PermMask[HiIndex]; 6140 if (Val < 0) 6141 continue; 6142 if (Val >= 4) 6143 break; 6144 } 6145 6146 Mask1[0] = PermMask[HiIndex]; 6147 Mask1[1] = -1; 6148 Mask1[2] = PermMask[HiIndex^1]; 6149 Mask1[3] = -1; 6150 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 6151 6152 if (HiIndex >= 2) { 6153 Mask1[0] = PermMask[0]; 6154 Mask1[1] = PermMask[1]; 6155 Mask1[2] = HiIndex & 1 ? 6 : 4; 6156 Mask1[3] = HiIndex & 1 ? 4 : 6; 6157 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 6158 } 6159 6160 Mask1[0] = HiIndex & 1 ? 2 : 0; 6161 Mask1[1] = HiIndex & 1 ? 0 : 2; 6162 Mask1[2] = PermMask[2]; 6163 Mask1[3] = PermMask[3]; 6164 if (Mask1[2] >= 0) 6165 Mask1[2] += 4; 6166 if (Mask1[3] >= 0) 6167 Mask1[3] += 4; 6168 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); 6169 } 6170 6171 // Break it into (shuffle shuffle_hi, shuffle_lo). 6172 int LoMask[] = { -1, -1, -1, -1 }; 6173 int HiMask[] = { -1, -1, -1, -1 }; 6174 6175 int *MaskPtr = LoMask; 6176 unsigned MaskIdx = 0; 6177 unsigned LoIdx = 0; 6178 unsigned HiIdx = 2; 6179 for (unsigned i = 0; i != 4; ++i) { 6180 if (i == 2) { 6181 MaskPtr = HiMask; 6182 MaskIdx = 1; 6183 LoIdx = 0; 6184 HiIdx = 2; 6185 } 6186 int Idx = PermMask[i]; 6187 if (Idx < 0) { 6188 Locs[i] = std::make_pair(-1, -1); 6189 } else if (Idx < 4) { 6190 Locs[i] = std::make_pair(MaskIdx, LoIdx); 6191 MaskPtr[LoIdx] = Idx; 6192 LoIdx++; 6193 } else { 6194 Locs[i] = std::make_pair(MaskIdx, HiIdx); 6195 MaskPtr[HiIdx] = Idx; 6196 HiIdx++; 6197 } 6198 } 6199 6200 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); 6201 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); 6202 int MaskOps[] = { -1, -1, -1, -1 }; 6203 for (unsigned i = 0; i != 4; ++i) 6204 if (Locs[i].first != -1) 6205 MaskOps[i] = Locs[i].first * 4 + Locs[i].second; 6206 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); 6207} 6208 6209static bool MayFoldVectorLoad(SDValue V) { 6210 if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) 6211 V = V.getOperand(0); 6212 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) 6213 V = V.getOperand(0); 6214 if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR && 6215 V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF) 6216 // BUILD_VECTOR (load), undef 6217 V = V.getOperand(0); 6218 if (MayFoldLoad(V)) 6219 return true; 6220 return false; 6221} 6222 6223// FIXME: the version above should always be used. Since there's 6224// a bug where several vector shuffles can't be folded because the 6225// DAG is not updated during lowering and a node claims to have two 6226// uses while it only has one, use this version, and let isel match 6227// another instruction if the load really happens to have more than 6228// one use. Remove this version after this bug get fixed. 6229// rdar://8434668, PR8156 6230static bool RelaxedMayFoldVectorLoad(SDValue V) { 6231 if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) 6232 V = V.getOperand(0); 6233 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) 6234 V = V.getOperand(0); 6235 if (ISD::isNormalLoad(V.getNode())) 6236 return true; 6237 return false; 6238} 6239 6240static 6241SDValue getMOVDDup(SDValue &Op, DebugLoc &dl, SDValue V1, SelectionDAG &DAG) { 6242 EVT VT = Op.getValueType(); 6243 6244 // Canonizalize to v2f64. 6245 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); 6246 return DAG.getNode(ISD::BITCAST, dl, VT, 6247 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64, 6248 V1, DAG)); 6249} 6250 6251static 6252SDValue getMOVLowToHigh(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, 6253 bool HasSSE2) { 6254 SDValue V1 = Op.getOperand(0); 6255 SDValue V2 = Op.getOperand(1); 6256 EVT VT = Op.getValueType(); 6257 6258 assert(VT != MVT::v2i64 && "unsupported shuffle type"); 6259 6260 if (HasSSE2 && VT == MVT::v2f64) 6261 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG); 6262 6263 // v4f32 or v4i32: canonizalized to v4f32 (which is legal for SSE1) 6264 return DAG.getNode(ISD::BITCAST, dl, VT, 6265 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32, 6266 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1), 6267 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG)); 6268} 6269 6270static 6271SDValue getMOVHighToLow(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG) { 6272 SDValue V1 = Op.getOperand(0); 6273 SDValue V2 = Op.getOperand(1); 6274 EVT VT = Op.getValueType(); 6275 6276 assert((VT == MVT::v4i32 || VT == MVT::v4f32) && 6277 "unsupported shuffle type"); 6278 6279 if (V2.getOpcode() == ISD::UNDEF) 6280 V2 = V1; 6281 6282 // v4i32 or v4f32 6283 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG); 6284} 6285 6286static 6287SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) { 6288 SDValue V1 = Op.getOperand(0); 6289 SDValue V2 = Op.getOperand(1); 6290 EVT VT = Op.getValueType(); 6291 unsigned NumElems = VT.getVectorNumElements(); 6292 6293 // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second 6294 // operand of these instructions is only memory, so check if there's a 6295 // potencial load folding here, otherwise use SHUFPS or MOVSD to match the 6296 // same masks. 6297 bool CanFoldLoad = false; 6298 6299 // Trivial case, when V2 comes from a load. 6300 if (MayFoldVectorLoad(V2)) 6301 CanFoldLoad = true; 6302 6303 // When V1 is a load, it can be folded later into a store in isel, example: 6304 // (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1) 6305 // turns into: 6306 // (MOVLPSmr addr:$src1, VR128:$src2) 6307 // So, recognize this potential and also use MOVLPS or MOVLPD 6308 else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op)) 6309 CanFoldLoad = true; 6310 6311 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 6312 if (CanFoldLoad) { 6313 if (HasSSE2 && NumElems == 2) 6314 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG); 6315 6316 if (NumElems == 4) 6317 // If we don't care about the second element, procede to use movss. 6318 if (SVOp->getMaskElt(1) != -1) 6319 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG); 6320 } 6321 6322 // movl and movlp will both match v2i64, but v2i64 is never matched by 6323 // movl earlier because we make it strict to avoid messing with the movlp load 6324 // folding logic (see the code above getMOVLP call). Match it here then, 6325 // this is horrible, but will stay like this until we move all shuffle 6326 // matching to x86 specific nodes. Note that for the 1st condition all 6327 // types are matched with movsd. 6328 if (HasSSE2) { 6329 // FIXME: isMOVLMask should be checked and matched before getMOVLP, 6330 // as to remove this logic from here, as much as possible 6331 if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT)) 6332 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); 6333 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); 6334 } 6335 6336 assert(VT != MVT::v4i32 && "unsupported shuffle type"); 6337 6338 // Invert the operand order and use SHUFPS to match it. 6339 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1, 6340 getShuffleSHUFImmediate(SVOp), DAG); 6341} 6342 6343SDValue 6344X86TargetLowering::NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG) const { 6345 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 6346 EVT VT = Op.getValueType(); 6347 DebugLoc dl = Op.getDebugLoc(); 6348 SDValue V1 = Op.getOperand(0); 6349 SDValue V2 = Op.getOperand(1); 6350 6351 if (isZeroShuffle(SVOp)) 6352 return getZeroVector(VT, Subtarget, DAG, dl); 6353 6354 // Handle splat operations 6355 if (SVOp->isSplat()) { 6356 unsigned NumElem = VT.getVectorNumElements(); 6357 int Size = VT.getSizeInBits(); 6358 6359 // Use vbroadcast whenever the splat comes from a foldable load 6360 SDValue Broadcast = LowerVectorBroadcast(Op, DAG); 6361 if (Broadcast.getNode()) 6362 return Broadcast; 6363 6364 // Handle splats by matching through known shuffle masks 6365 if ((Size == 128 && NumElem <= 4) || 6366 (Size == 256 && NumElem < 8)) 6367 return SDValue(); 6368 6369 // All remaning splats are promoted to target supported vector shuffles. 6370 return PromoteSplat(SVOp, DAG); 6371 } 6372 6373 // If the shuffle can be profitably rewritten as a narrower shuffle, then 6374 // do it! 6375 if (VT == MVT::v8i16 || VT == MVT::v16i8 || 6376 VT == MVT::v16i16 || VT == MVT::v32i8) { 6377 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 6378 if (NewOp.getNode()) 6379 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp); 6380 } else if ((VT == MVT::v4i32 || 6381 (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 6382 // FIXME: Figure out a cleaner way to do this. 6383 // Try to make use of movq to zero out the top part. 6384 if (ISD::isBuildVectorAllZeros(V2.getNode())) { 6385 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 6386 if (NewOp.getNode()) { 6387 EVT NewVT = NewOp.getValueType(); 6388 if (isCommutedMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(), 6389 NewVT, true, false)) 6390 return getVZextMovL(VT, NewVT, NewOp.getOperand(0), 6391 DAG, Subtarget, dl); 6392 } 6393 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { 6394 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); 6395 if (NewOp.getNode()) { 6396 EVT NewVT = NewOp.getValueType(); 6397 if (isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(), NewVT)) 6398 return getVZextMovL(VT, NewVT, NewOp.getOperand(1), 6399 DAG, Subtarget, dl); 6400 } 6401 } 6402 } 6403 return SDValue(); 6404} 6405 6406SDValue 6407X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { 6408 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 6409 SDValue V1 = Op.getOperand(0); 6410 SDValue V2 = Op.getOperand(1); 6411 EVT VT = Op.getValueType(); 6412 DebugLoc dl = Op.getDebugLoc(); 6413 unsigned NumElems = VT.getVectorNumElements(); 6414 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 6415 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 6416 bool V1IsSplat = false; 6417 bool V2IsSplat = false; 6418 bool HasSSE2 = Subtarget->hasSSE2(); 6419 bool HasAVX = Subtarget->hasAVX(); 6420 bool HasAVX2 = Subtarget->hasAVX2(); 6421 MachineFunction &MF = DAG.getMachineFunction(); 6422 bool OptForSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 6423 6424 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles"); 6425 6426 if (V1IsUndef && V2IsUndef) 6427 return DAG.getUNDEF(VT); 6428 6429 assert(!V1IsUndef && "Op 1 of shuffle should not be undef"); 6430 6431 // Vector shuffle lowering takes 3 steps: 6432 // 6433 // 1) Normalize the input vectors. Here splats, zeroed vectors, profitable 6434 // narrowing and commutation of operands should be handled. 6435 // 2) Matching of shuffles with known shuffle masks to x86 target specific 6436 // shuffle nodes. 6437 // 3) Rewriting of unmatched masks into new generic shuffle operations, 6438 // so the shuffle can be broken into other shuffles and the legalizer can 6439 // try the lowering again. 6440 // 6441 // The general idea is that no vector_shuffle operation should be left to 6442 // be matched during isel, all of them must be converted to a target specific 6443 // node here. 6444 6445 // Normalize the input vectors. Here splats, zeroed vectors, profitable 6446 // narrowing and commutation of operands should be handled. The actual code 6447 // doesn't include all of those, work in progress... 6448 SDValue NewOp = NormalizeVectorShuffle(Op, DAG); 6449 if (NewOp.getNode()) 6450 return NewOp; 6451 6452 SmallVector<int, 8> M(SVOp->getMask().begin(), SVOp->getMask().end()); 6453 6454 // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and 6455 // unpckh_undef). Only use pshufd if speed is more important than size. 6456 if (OptForSize && isUNPCKL_v_undef_Mask(M, VT, HasAVX2)) 6457 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); 6458 if (OptForSize && isUNPCKH_v_undef_Mask(M, VT, HasAVX2)) 6459 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); 6460 6461 if (isMOVDDUPMask(M, VT) && Subtarget->hasSSE3() && 6462 V2IsUndef && RelaxedMayFoldVectorLoad(V1)) 6463 return getMOVDDup(Op, dl, V1, DAG); 6464 6465 if (isMOVHLPS_v_undef_Mask(M, VT)) 6466 return getMOVHighToLow(Op, dl, DAG); 6467 6468 // Use to match splats 6469 if (HasSSE2 && isUNPCKHMask(M, VT, HasAVX2) && V2IsUndef && 6470 (VT == MVT::v2f64 || VT == MVT::v2i64)) 6471 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); 6472 6473 if (isPSHUFDMask(M, VT)) { 6474 // The actual implementation will match the mask in the if above and then 6475 // during isel it can match several different instructions, not only pshufd 6476 // as its name says, sad but true, emulate the behavior for now... 6477 if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64))) 6478 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG); 6479 6480 unsigned TargetMask = getShuffleSHUFImmediate(SVOp); 6481 6482 if (HasAVX && (VT == MVT::v4f32 || VT == MVT::v2f64)) 6483 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1, TargetMask, DAG); 6484 6485 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32)) 6486 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG); 6487 6488 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1, 6489 TargetMask, DAG); 6490 } 6491 6492 // Check if this can be converted into a logical shift. 6493 bool isLeft = false; 6494 unsigned ShAmt = 0; 6495 SDValue ShVal; 6496 bool isShift = HasSSE2 && isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt); 6497 if (isShift && ShVal.hasOneUse()) { 6498 // If the shifted value has multiple uses, it may be cheaper to use 6499 // v_set0 + movlhps or movhlps, etc. 6500 EVT EltVT = VT.getVectorElementType(); 6501 ShAmt *= EltVT.getSizeInBits(); 6502 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 6503 } 6504 6505 if (isMOVLMask(M, VT)) { 6506 if (ISD::isBuildVectorAllZeros(V1.getNode())) 6507 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl); 6508 if (!isMOVLPMask(M, VT)) { 6509 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64)) 6510 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); 6511 6512 if (VT == MVT::v4i32 || VT == MVT::v4f32) 6513 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); 6514 } 6515 } 6516 6517 // FIXME: fold these into legal mask. 6518 if (isMOVLHPSMask(M, VT) && !isUNPCKLMask(M, VT, HasAVX2)) 6519 return getMOVLowToHigh(Op, dl, DAG, HasSSE2); 6520 6521 if (isMOVHLPSMask(M, VT)) 6522 return getMOVHighToLow(Op, dl, DAG); 6523 6524 if (V2IsUndef && isMOVSHDUPMask(M, VT, Subtarget)) 6525 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG); 6526 6527 if (V2IsUndef && isMOVSLDUPMask(M, VT, Subtarget)) 6528 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG); 6529 6530 if (isMOVLPMask(M, VT)) 6531 return getMOVLP(Op, dl, DAG, HasSSE2); 6532 6533 if (ShouldXformToMOVHLPS(M, VT) || 6534 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), M, VT)) 6535 return CommuteVectorShuffle(SVOp, DAG); 6536 6537 if (isShift) { 6538 // No better options. Use a vshldq / vsrldq. 6539 EVT EltVT = VT.getVectorElementType(); 6540 ShAmt *= EltVT.getSizeInBits(); 6541 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 6542 } 6543 6544 bool Commuted = false; 6545 // FIXME: This should also accept a bitcast of a splat? Be careful, not 6546 // 1,1,1,1 -> v8i16 though. 6547 V1IsSplat = isSplatVector(V1.getNode()); 6548 V2IsSplat = isSplatVector(V2.getNode()); 6549 6550 // Canonicalize the splat or undef, if present, to be on the RHS. 6551 if (!V2IsUndef && V1IsSplat && !V2IsSplat) { 6552 CommuteVectorShuffleMask(M, NumElems); 6553 std::swap(V1, V2); 6554 std::swap(V1IsSplat, V2IsSplat); 6555 Commuted = true; 6556 } 6557 6558 if (isCommutedMOVLMask(M, VT, V2IsSplat, V2IsUndef)) { 6559 // Shuffling low element of v1 into undef, just return v1. 6560 if (V2IsUndef) 6561 return V1; 6562 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which 6563 // the instruction selector will not match, so get a canonical MOVL with 6564 // swapped operands to undo the commute. 6565 return getMOVL(DAG, dl, VT, V2, V1); 6566 } 6567 6568 if (isUNPCKLMask(M, VT, HasAVX2)) 6569 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); 6570 6571 if (isUNPCKHMask(M, VT, HasAVX2)) 6572 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); 6573 6574 if (V2IsSplat) { 6575 // Normalize mask so all entries that point to V2 points to its first 6576 // element then try to match unpck{h|l} again. If match, return a 6577 // new vector_shuffle with the corrected mask.p 6578 SmallVector<int, 8> NewMask(M.begin(), M.end()); 6579 NormalizeMask(NewMask, NumElems); 6580 if (isUNPCKLMask(NewMask, VT, HasAVX2, true)) 6581 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); 6582 if (isUNPCKHMask(NewMask, VT, HasAVX2, true)) 6583 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); 6584 } 6585 6586 if (Commuted) { 6587 // Commute is back and try unpck* again. 6588 // FIXME: this seems wrong. 6589 CommuteVectorShuffleMask(M, NumElems); 6590 std::swap(V1, V2); 6591 std::swap(V1IsSplat, V2IsSplat); 6592 Commuted = false; 6593 6594 if (isUNPCKLMask(M, VT, HasAVX2)) 6595 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); 6596 6597 if (isUNPCKHMask(M, VT, HasAVX2)) 6598 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); 6599 } 6600 6601 // Normalize the node to match x86 shuffle ops if needed 6602 if (!V2IsUndef && (isSHUFPMask(M, VT, HasAVX, /* Commuted */ true))) 6603 return CommuteVectorShuffle(SVOp, DAG); 6604 6605 // The checks below are all present in isShuffleMaskLegal, but they are 6606 // inlined here right now to enable us to directly emit target specific 6607 // nodes, and remove one by one until they don't return Op anymore. 6608 6609 if (isPALIGNRMask(M, VT, Subtarget)) 6610 return getTargetShuffleNode(X86ISD::PALIGN, dl, VT, V1, V2, 6611 getShufflePALIGNRImmediate(SVOp), 6612 DAG); 6613 6614 if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) && 6615 SVOp->getSplatIndex() == 0 && V2IsUndef) { 6616 if (VT == MVT::v2f64 || VT == MVT::v2i64) 6617 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); 6618 } 6619 6620 if (isPSHUFHWMask(M, VT, HasAVX2)) 6621 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1, 6622 getShufflePSHUFHWImmediate(SVOp), 6623 DAG); 6624 6625 if (isPSHUFLWMask(M, VT, HasAVX2)) 6626 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1, 6627 getShufflePSHUFLWImmediate(SVOp), 6628 DAG); 6629 6630 if (isSHUFPMask(M, VT, HasAVX)) 6631 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2, 6632 getShuffleSHUFImmediate(SVOp), DAG); 6633 6634 if (isUNPCKL_v_undef_Mask(M, VT, HasAVX2)) 6635 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); 6636 if (isUNPCKH_v_undef_Mask(M, VT, HasAVX2)) 6637 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); 6638 6639 //===--------------------------------------------------------------------===// 6640 // Generate target specific nodes for 128 or 256-bit shuffles only 6641 // supported in the AVX instruction set. 6642 // 6643 6644 // Handle VMOVDDUPY permutations 6645 if (V2IsUndef && isMOVDDUPYMask(M, VT, HasAVX)) 6646 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG); 6647 6648 // Handle VPERMILPS/D* permutations 6649 if (isVPERMILPMask(M, VT, HasAVX)) { 6650 if (HasAVX2 && VT == MVT::v8i32) 6651 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, 6652 getShuffleSHUFImmediate(SVOp), DAG); 6653 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1, 6654 getShuffleSHUFImmediate(SVOp), DAG); 6655 } 6656 6657 // Handle VPERM2F128/VPERM2I128 permutations 6658 if (isVPERM2X128Mask(M, VT, HasAVX)) 6659 return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1, 6660 V2, getShuffleVPERM2X128Immediate(SVOp), DAG); 6661 6662 SDValue BlendOp = LowerVECTOR_SHUFFLEtoBlend(SVOp, Subtarget, DAG); 6663 if (BlendOp.getNode()) 6664 return BlendOp; 6665 6666 if (V2IsUndef && HasAVX2 && (VT == MVT::v8i32 || VT == MVT::v8f32)) { 6667 SmallVector<SDValue, 8> permclMask; 6668 for (unsigned i = 0; i != 8; ++i) { 6669 permclMask.push_back(DAG.getConstant((M[i]>=0) ? M[i] : 0, MVT::i32)); 6670 } 6671 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, 6672 &permclMask[0], 8); 6673 // Bitcast is for VPERMPS since mask is v8i32 but node takes v8f32 6674 return DAG.getNode(X86ISD::VPERMV, dl, VT, 6675 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1); 6676 } 6677 6678 if (V2IsUndef && HasAVX2 && (VT == MVT::v4i64 || VT == MVT::v4f64)) 6679 return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1, 6680 getShuffleCLImmediate(SVOp), DAG); 6681 6682 6683 //===--------------------------------------------------------------------===// 6684 // Since no target specific shuffle was selected for this generic one, 6685 // lower it into other known shuffles. FIXME: this isn't true yet, but 6686 // this is the plan. 6687 // 6688 6689 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 6690 if (VT == MVT::v8i16) { 6691 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, DAG); 6692 if (NewOp.getNode()) 6693 return NewOp; 6694 } 6695 6696 if (VT == MVT::v16i8) { 6697 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this); 6698 if (NewOp.getNode()) 6699 return NewOp; 6700 } 6701 6702 // Handle all 128-bit wide vectors with 4 elements, and match them with 6703 // several different shuffle types. 6704 if (NumElems == 4 && VT.getSizeInBits() == 128) 6705 return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG); 6706 6707 // Handle general 256-bit shuffles 6708 if (VT.is256BitVector()) 6709 return LowerVECTOR_SHUFFLE_256(SVOp, DAG); 6710 6711 return SDValue(); 6712} 6713 6714SDValue 6715X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 6716 SelectionDAG &DAG) const { 6717 EVT VT = Op.getValueType(); 6718 DebugLoc dl = Op.getDebugLoc(); 6719 6720 if (Op.getOperand(0).getValueType().getSizeInBits() != 128) 6721 return SDValue(); 6722 6723 if (VT.getSizeInBits() == 8) { 6724 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, 6725 Op.getOperand(0), Op.getOperand(1)); 6726 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 6727 DAG.getValueType(VT)); 6728 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6729 } 6730 6731 if (VT.getSizeInBits() == 16) { 6732 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6733 // If Idx is 0, it's cheaper to do a move instead of a pextrw. 6734 if (Idx == 0) 6735 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 6736 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6737 DAG.getNode(ISD::BITCAST, dl, 6738 MVT::v4i32, 6739 Op.getOperand(0)), 6740 Op.getOperand(1))); 6741 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, 6742 Op.getOperand(0), Op.getOperand(1)); 6743 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 6744 DAG.getValueType(VT)); 6745 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6746 } 6747 6748 if (VT == MVT::f32) { 6749 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 6750 // the result back to FR32 register. It's only worth matching if the 6751 // result has a single use which is a store or a bitcast to i32. And in 6752 // the case of a store, it's not worth it if the index is a constant 0, 6753 // because a MOVSSmr can be used instead, which is smaller and faster. 6754 if (!Op.hasOneUse()) 6755 return SDValue(); 6756 SDNode *User = *Op.getNode()->use_begin(); 6757 if ((User->getOpcode() != ISD::STORE || 6758 (isa<ConstantSDNode>(Op.getOperand(1)) && 6759 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) && 6760 (User->getOpcode() != ISD::BITCAST || 6761 User->getValueType(0) != MVT::i32)) 6762 return SDValue(); 6763 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6764 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, 6765 Op.getOperand(0)), 6766 Op.getOperand(1)); 6767 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); 6768 } 6769 6770 if (VT == MVT::i32 || VT == MVT::i64) { 6771 // ExtractPS/pextrq works with constant index. 6772 if (isa<ConstantSDNode>(Op.getOperand(1))) 6773 return Op; 6774 } 6775 return SDValue(); 6776} 6777 6778 6779SDValue 6780X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 6781 SelectionDAG &DAG) const { 6782 if (!isa<ConstantSDNode>(Op.getOperand(1))) 6783 return SDValue(); 6784 6785 SDValue Vec = Op.getOperand(0); 6786 EVT VecVT = Vec.getValueType(); 6787 6788 // If this is a 256-bit vector result, first extract the 128-bit vector and 6789 // then extract the element from the 128-bit vector. 6790 if (VecVT.getSizeInBits() == 256) { 6791 DebugLoc dl = Op.getNode()->getDebugLoc(); 6792 unsigned NumElems = VecVT.getVectorNumElements(); 6793 SDValue Idx = Op.getOperand(1); 6794 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 6795 6796 // Get the 128-bit vector. 6797 Vec = Extract128BitVector(Vec, IdxVal, DAG, dl); 6798 6799 if (IdxVal >= NumElems/2) 6800 IdxVal -= NumElems/2; 6801 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, 6802 DAG.getConstant(IdxVal, MVT::i32)); 6803 } 6804 6805 assert(Vec.getValueSizeInBits() <= 128 && "Unexpected vector length"); 6806 6807 if (Subtarget->hasSSE41()) { 6808 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 6809 if (Res.getNode()) 6810 return Res; 6811 } 6812 6813 EVT VT = Op.getValueType(); 6814 DebugLoc dl = Op.getDebugLoc(); 6815 // TODO: handle v16i8. 6816 if (VT.getSizeInBits() == 16) { 6817 SDValue Vec = Op.getOperand(0); 6818 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6819 if (Idx == 0) 6820 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 6821 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 6822 DAG.getNode(ISD::BITCAST, dl, 6823 MVT::v4i32, Vec), 6824 Op.getOperand(1))); 6825 // Transform it so it match pextrw which produces a 32-bit result. 6826 EVT EltVT = MVT::i32; 6827 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT, 6828 Op.getOperand(0), Op.getOperand(1)); 6829 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract, 6830 DAG.getValueType(VT)); 6831 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 6832 } 6833 6834 if (VT.getSizeInBits() == 32) { 6835 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6836 if (Idx == 0) 6837 return Op; 6838 6839 // SHUFPS the element to the lowest double word, then movss. 6840 int Mask[4] = { static_cast<int>(Idx), -1, -1, -1 }; 6841 EVT VVT = Op.getOperand(0).getValueType(); 6842 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 6843 DAG.getUNDEF(VVT), Mask); 6844 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 6845 DAG.getIntPtrConstant(0)); 6846 } 6847 6848 if (VT.getSizeInBits() == 64) { 6849 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 6850 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 6851 // to match extract_elt for f64. 6852 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 6853 if (Idx == 0) 6854 return Op; 6855 6856 // UNPCKHPD the element to the lowest double word, then movsd. 6857 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 6858 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 6859 int Mask[2] = { 1, -1 }; 6860 EVT VVT = Op.getOperand(0).getValueType(); 6861 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 6862 DAG.getUNDEF(VVT), Mask); 6863 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 6864 DAG.getIntPtrConstant(0)); 6865 } 6866 6867 return SDValue(); 6868} 6869 6870SDValue 6871X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, 6872 SelectionDAG &DAG) const { 6873 EVT VT = Op.getValueType(); 6874 EVT EltVT = VT.getVectorElementType(); 6875 DebugLoc dl = Op.getDebugLoc(); 6876 6877 SDValue N0 = Op.getOperand(0); 6878 SDValue N1 = Op.getOperand(1); 6879 SDValue N2 = Op.getOperand(2); 6880 6881 if (VT.getSizeInBits() == 256) 6882 return SDValue(); 6883 6884 if ((EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) && 6885 isa<ConstantSDNode>(N2)) { 6886 unsigned Opc; 6887 if (VT == MVT::v8i16) 6888 Opc = X86ISD::PINSRW; 6889 else if (VT == MVT::v16i8) 6890 Opc = X86ISD::PINSRB; 6891 else 6892 Opc = X86ISD::PINSRB; 6893 6894 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 6895 // argument. 6896 if (N1.getValueType() != MVT::i32) 6897 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 6898 if (N2.getValueType() != MVT::i32) 6899 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 6900 return DAG.getNode(Opc, dl, VT, N0, N1, N2); 6901 } 6902 6903 if (EltVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 6904 // Bits [7:6] of the constant are the source select. This will always be 6905 // zero here. The DAG Combiner may combine an extract_elt index into these 6906 // bits. For example (insert (extract, 3), 2) could be matched by putting 6907 // the '3' into bits [7:6] of X86ISD::INSERTPS. 6908 // Bits [5:4] of the constant are the destination select. This is the 6909 // value of the incoming immediate. 6910 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 6911 // combine either bitwise AND or insert of float 0.0 to set these bits. 6912 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4); 6913 // Create this as a scalar to vector.. 6914 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); 6915 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2); 6916 } 6917 6918 if ((EltVT == MVT::i32 || EltVT == MVT::i64) && isa<ConstantSDNode>(N2)) { 6919 // PINSR* works with constant index. 6920 return Op; 6921 } 6922 return SDValue(); 6923} 6924 6925SDValue 6926X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { 6927 EVT VT = Op.getValueType(); 6928 EVT EltVT = VT.getVectorElementType(); 6929 6930 DebugLoc dl = Op.getDebugLoc(); 6931 SDValue N0 = Op.getOperand(0); 6932 SDValue N1 = Op.getOperand(1); 6933 SDValue N2 = Op.getOperand(2); 6934 6935 // If this is a 256-bit vector result, first extract the 128-bit vector, 6936 // insert the element into the extracted half and then place it back. 6937 if (VT.getSizeInBits() == 256) { 6938 if (!isa<ConstantSDNode>(N2)) 6939 return SDValue(); 6940 6941 // Get the desired 128-bit vector half. 6942 unsigned NumElems = VT.getVectorNumElements(); 6943 unsigned IdxVal = cast<ConstantSDNode>(N2)->getZExtValue(); 6944 SDValue V = Extract128BitVector(N0, IdxVal, DAG, dl); 6945 6946 // Insert the element into the desired half. 6947 bool Upper = IdxVal >= NumElems/2; 6948 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1, 6949 DAG.getConstant(Upper ? IdxVal-NumElems/2 : IdxVal, MVT::i32)); 6950 6951 // Insert the changed part back to the 256-bit vector 6952 return Insert128BitVector(N0, V, IdxVal, DAG, dl); 6953 } 6954 6955 if (Subtarget->hasSSE41()) 6956 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 6957 6958 if (EltVT == MVT::i8) 6959 return SDValue(); 6960 6961 if (EltVT.getSizeInBits() == 16 && isa<ConstantSDNode>(N2)) { 6962 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 6963 // as its second argument. 6964 if (N1.getValueType() != MVT::i32) 6965 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 6966 if (N2.getValueType() != MVT::i32) 6967 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 6968 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2); 6969 } 6970 return SDValue(); 6971} 6972 6973SDValue 6974X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const { 6975 LLVMContext *Context = DAG.getContext(); 6976 DebugLoc dl = Op.getDebugLoc(); 6977 EVT OpVT = Op.getValueType(); 6978 6979 // If this is a 256-bit vector result, first insert into a 128-bit 6980 // vector and then insert into the 256-bit vector. 6981 if (OpVT.getSizeInBits() > 128) { 6982 // Insert into a 128-bit vector. 6983 EVT VT128 = EVT::getVectorVT(*Context, 6984 OpVT.getVectorElementType(), 6985 OpVT.getVectorNumElements() / 2); 6986 6987 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0)); 6988 6989 // Insert the 128-bit vector. 6990 return Insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl); 6991 } 6992 6993 if (OpVT == MVT::v1i64 && 6994 Op.getOperand(0).getValueType() == MVT::i64) 6995 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); 6996 6997 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); 6998 assert(OpVT.getSizeInBits() == 128 && "Expected an SSE type!"); 6999 return DAG.getNode(ISD::BITCAST, dl, OpVT, 7000 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); 7001} 7002 7003// Lower a node with an EXTRACT_SUBVECTOR opcode. This may result in 7004// a simple subregister reference or explicit instructions to grab 7005// upper bits of a vector. 7006SDValue 7007X86TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { 7008 if (Subtarget->hasAVX()) { 7009 DebugLoc dl = Op.getNode()->getDebugLoc(); 7010 SDValue Vec = Op.getNode()->getOperand(0); 7011 SDValue Idx = Op.getNode()->getOperand(1); 7012 7013 if (Op.getNode()->getValueType(0).getSizeInBits() == 128 && 7014 Vec.getNode()->getValueType(0).getSizeInBits() == 256 && 7015 isa<ConstantSDNode>(Idx)) { 7016 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 7017 return Extract128BitVector(Vec, IdxVal, DAG, dl); 7018 } 7019 } 7020 return SDValue(); 7021} 7022 7023// Lower a node with an INSERT_SUBVECTOR opcode. This may result in a 7024// simple superregister reference or explicit instructions to insert 7025// the upper bits of a vector. 7026SDValue 7027X86TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { 7028 if (Subtarget->hasAVX()) { 7029 DebugLoc dl = Op.getNode()->getDebugLoc(); 7030 SDValue Vec = Op.getNode()->getOperand(0); 7031 SDValue SubVec = Op.getNode()->getOperand(1); 7032 SDValue Idx = Op.getNode()->getOperand(2); 7033 7034 if (Op.getNode()->getValueType(0).getSizeInBits() == 256 && 7035 SubVec.getNode()->getValueType(0).getSizeInBits() == 128 && 7036 isa<ConstantSDNode>(Idx)) { 7037 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 7038 return Insert128BitVector(Vec, SubVec, IdxVal, DAG, dl); 7039 } 7040 } 7041 return SDValue(); 7042} 7043 7044// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 7045// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 7046// one of the above mentioned nodes. It has to be wrapped because otherwise 7047// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 7048// be used to form addressing mode. These wrapped nodes will be selected 7049// into MOV32ri. 7050SDValue 7051X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { 7052 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 7053 7054 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 7055 // global base reg. 7056 unsigned char OpFlag = 0; 7057 unsigned WrapperKind = X86ISD::Wrapper; 7058 CodeModel::Model M = getTargetMachine().getCodeModel(); 7059 7060 if (Subtarget->isPICStyleRIPRel() && 7061 (M == CodeModel::Small || M == CodeModel::Kernel)) 7062 WrapperKind = X86ISD::WrapperRIP; 7063 else if (Subtarget->isPICStyleGOT()) 7064 OpFlag = X86II::MO_GOTOFF; 7065 else if (Subtarget->isPICStyleStubPIC()) 7066 OpFlag = X86II::MO_PIC_BASE_OFFSET; 7067 7068 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(), 7069 CP->getAlignment(), 7070 CP->getOffset(), OpFlag); 7071 DebugLoc DL = CP->getDebugLoc(); 7072 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 7073 // With PIC, the address is actually $g + Offset. 7074 if (OpFlag) { 7075 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 7076 DAG.getNode(X86ISD::GlobalBaseReg, 7077 DebugLoc(), getPointerTy()), 7078 Result); 7079 } 7080 7081 return Result; 7082} 7083 7084SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 7085 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 7086 7087 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 7088 // global base reg. 7089 unsigned char OpFlag = 0; 7090 unsigned WrapperKind = X86ISD::Wrapper; 7091 CodeModel::Model M = getTargetMachine().getCodeModel(); 7092 7093 if (Subtarget->isPICStyleRIPRel() && 7094 (M == CodeModel::Small || M == CodeModel::Kernel)) 7095 WrapperKind = X86ISD::WrapperRIP; 7096 else if (Subtarget->isPICStyleGOT()) 7097 OpFlag = X86II::MO_GOTOFF; 7098 else if (Subtarget->isPICStyleStubPIC()) 7099 OpFlag = X86II::MO_PIC_BASE_OFFSET; 7100 7101 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(), 7102 OpFlag); 7103 DebugLoc DL = JT->getDebugLoc(); 7104 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 7105 7106 // With PIC, the address is actually $g + Offset. 7107 if (OpFlag) 7108 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 7109 DAG.getNode(X86ISD::GlobalBaseReg, 7110 DebugLoc(), getPointerTy()), 7111 Result); 7112 7113 return Result; 7114} 7115 7116SDValue 7117X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const { 7118 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 7119 7120 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 7121 // global base reg. 7122 unsigned char OpFlag = 0; 7123 unsigned WrapperKind = X86ISD::Wrapper; 7124 CodeModel::Model M = getTargetMachine().getCodeModel(); 7125 7126 if (Subtarget->isPICStyleRIPRel() && 7127 (M == CodeModel::Small || M == CodeModel::Kernel)) { 7128 if (Subtarget->isTargetDarwin() || Subtarget->isTargetELF()) 7129 OpFlag = X86II::MO_GOTPCREL; 7130 WrapperKind = X86ISD::WrapperRIP; 7131 } else if (Subtarget->isPICStyleGOT()) { 7132 OpFlag = X86II::MO_GOT; 7133 } else if (Subtarget->isPICStyleStubPIC()) { 7134 OpFlag = X86II::MO_DARWIN_NONLAZY_PIC_BASE; 7135 } else if (Subtarget->isPICStyleStubNoDynamic()) { 7136 OpFlag = X86II::MO_DARWIN_NONLAZY; 7137 } 7138 7139 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag); 7140 7141 DebugLoc DL = Op.getDebugLoc(); 7142 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 7143 7144 7145 // With PIC, the address is actually $g + Offset. 7146 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 7147 !Subtarget->is64Bit()) { 7148 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 7149 DAG.getNode(X86ISD::GlobalBaseReg, 7150 DebugLoc(), getPointerTy()), 7151 Result); 7152 } 7153 7154 // For symbols that require a load from a stub to get the address, emit the 7155 // load. 7156 if (isGlobalStubReference(OpFlag)) 7157 Result = DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), Result, 7158 MachinePointerInfo::getGOT(), false, false, false, 0); 7159 7160 return Result; 7161} 7162 7163SDValue 7164X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { 7165 // Create the TargetBlockAddressAddress node. 7166 unsigned char OpFlags = 7167 Subtarget->ClassifyBlockAddressReference(); 7168 CodeModel::Model M = getTargetMachine().getCodeModel(); 7169 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 7170 DebugLoc dl = Op.getDebugLoc(); 7171 SDValue Result = DAG.getBlockAddress(BA, getPointerTy(), 7172 /*isTarget=*/true, OpFlags); 7173 7174 if (Subtarget->isPICStyleRIPRel() && 7175 (M == CodeModel::Small || M == CodeModel::Kernel)) 7176 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); 7177 else 7178 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); 7179 7180 // With PIC, the address is actually $g + Offset. 7181 if (isGlobalRelativeToPICBase(OpFlags)) { 7182 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), 7183 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), 7184 Result); 7185 } 7186 7187 return Result; 7188} 7189 7190SDValue 7191X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl, 7192 int64_t Offset, 7193 SelectionDAG &DAG) const { 7194 // Create the TargetGlobalAddress node, folding in the constant 7195 // offset if it is legal. 7196 unsigned char OpFlags = 7197 Subtarget->ClassifyGlobalReference(GV, getTargetMachine()); 7198 CodeModel::Model M = getTargetMachine().getCodeModel(); 7199 SDValue Result; 7200 if (OpFlags == X86II::MO_NO_FLAG && 7201 X86::isOffsetSuitableForCodeModel(Offset, M)) { 7202 // A direct static reference to a global. 7203 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset); 7204 Offset = 0; 7205 } else { 7206 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags); 7207 } 7208 7209 if (Subtarget->isPICStyleRIPRel() && 7210 (M == CodeModel::Small || M == CodeModel::Kernel)) 7211 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); 7212 else 7213 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); 7214 7215 // With PIC, the address is actually $g + Offset. 7216 if (isGlobalRelativeToPICBase(OpFlags)) { 7217 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), 7218 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), 7219 Result); 7220 } 7221 7222 // For globals that require a load from a stub to get the address, emit the 7223 // load. 7224 if (isGlobalStubReference(OpFlags)) 7225 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result, 7226 MachinePointerInfo::getGOT(), false, false, false, 0); 7227 7228 // If there was a non-zero offset that we didn't fold, create an explicit 7229 // addition for it. 7230 if (Offset != 0) 7231 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result, 7232 DAG.getConstant(Offset, getPointerTy())); 7233 7234 return Result; 7235} 7236 7237SDValue 7238X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { 7239 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 7240 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset(); 7241 return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG); 7242} 7243 7244static SDValue 7245GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, 7246 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, 7247 unsigned char OperandFlags) { 7248 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 7249 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 7250 DebugLoc dl = GA->getDebugLoc(); 7251 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, 7252 GA->getValueType(0), 7253 GA->getOffset(), 7254 OperandFlags); 7255 if (InFlag) { 7256 SDValue Ops[] = { Chain, TGA, *InFlag }; 7257 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3); 7258 } else { 7259 SDValue Ops[] = { Chain, TGA }; 7260 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2); 7261 } 7262 7263 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls. 7264 MFI->setAdjustsStack(true); 7265 7266 SDValue Flag = Chain.getValue(1); 7267 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag); 7268} 7269 7270// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 7271static SDValue 7272LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 7273 const EVT PtrVT) { 7274 SDValue InFlag; 7275 DebugLoc dl = GA->getDebugLoc(); // ? function entry point might be better 7276 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, 7277 DAG.getNode(X86ISD::GlobalBaseReg, 7278 DebugLoc(), PtrVT), InFlag); 7279 InFlag = Chain.getValue(1); 7280 7281 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD); 7282} 7283 7284// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 7285static SDValue 7286LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 7287 const EVT PtrVT) { 7288 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT, 7289 X86::RAX, X86II::MO_TLSGD); 7290} 7291 7292// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 7293// "local exec" model. 7294static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 7295 const EVT PtrVT, TLSModel::Model model, 7296 bool is64Bit) { 7297 DebugLoc dl = GA->getDebugLoc(); 7298 7299 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit). 7300 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(), 7301 is64Bit ? 257 : 256)); 7302 7303 SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 7304 DAG.getIntPtrConstant(0), 7305 MachinePointerInfo(Ptr), 7306 false, false, false, 0); 7307 7308 unsigned char OperandFlags = 0; 7309 // Most TLS accesses are not RIP relative, even on x86-64. One exception is 7310 // initialexec. 7311 unsigned WrapperKind = X86ISD::Wrapper; 7312 if (model == TLSModel::LocalExec) { 7313 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF; 7314 } else if (is64Bit) { 7315 assert(model == TLSModel::InitialExec); 7316 OperandFlags = X86II::MO_GOTTPOFF; 7317 WrapperKind = X86ISD::WrapperRIP; 7318 } else { 7319 assert(model == TLSModel::InitialExec); 7320 OperandFlags = X86II::MO_INDNTPOFF; 7321 } 7322 7323 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 7324 // exec) 7325 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, 7326 GA->getValueType(0), 7327 GA->getOffset(), OperandFlags); 7328 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); 7329 7330 if (model == TLSModel::InitialExec) 7331 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset, 7332 MachinePointerInfo::getGOT(), false, false, false, 0); 7333 7334 // The address of the thread local variable is the add of the thread 7335 // pointer with the offset of the variable. 7336 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); 7337} 7338 7339SDValue 7340X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { 7341 7342 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 7343 const GlobalValue *GV = GA->getGlobal(); 7344 7345 if (Subtarget->isTargetELF()) { 7346 // TODO: implement the "local dynamic" model 7347 // TODO: implement the "initial exec"model for pic executables 7348 7349 // If GV is an alias then use the aliasee for determining 7350 // thread-localness. 7351 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 7352 GV = GA->resolveAliasedGlobal(false); 7353 7354 TLSModel::Model model = getTargetMachine().getTLSModel(GV); 7355 7356 switch (model) { 7357 case TLSModel::GeneralDynamic: 7358 case TLSModel::LocalDynamic: // not implemented 7359 if (Subtarget->is64Bit()) 7360 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 7361 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 7362 7363 case TLSModel::InitialExec: 7364 case TLSModel::LocalExec: 7365 return LowerToTLSExecModel(GA, DAG, getPointerTy(), model, 7366 Subtarget->is64Bit()); 7367 } 7368 llvm_unreachable("Unknown TLS model."); 7369 } 7370 7371 if (Subtarget->isTargetDarwin()) { 7372 // Darwin only has one model of TLS. Lower to that. 7373 unsigned char OpFlag = 0; 7374 unsigned WrapperKind = Subtarget->isPICStyleRIPRel() ? 7375 X86ISD::WrapperRIP : X86ISD::Wrapper; 7376 7377 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 7378 // global base reg. 7379 bool PIC32 = (getTargetMachine().getRelocationModel() == Reloc::PIC_) && 7380 !Subtarget->is64Bit(); 7381 if (PIC32) 7382 OpFlag = X86II::MO_TLVP_PIC_BASE; 7383 else 7384 OpFlag = X86II::MO_TLVP; 7385 DebugLoc DL = Op.getDebugLoc(); 7386 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, 7387 GA->getValueType(0), 7388 GA->getOffset(), OpFlag); 7389 SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 7390 7391 // With PIC32, the address is actually $g + Offset. 7392 if (PIC32) 7393 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(), 7394 DAG.getNode(X86ISD::GlobalBaseReg, 7395 DebugLoc(), getPointerTy()), 7396 Offset); 7397 7398 // Lowering the machine isd will make sure everything is in the right 7399 // location. 7400 SDValue Chain = DAG.getEntryNode(); 7401 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 7402 SDValue Args[] = { Chain, Offset }; 7403 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args, 2); 7404 7405 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls. 7406 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 7407 MFI->setAdjustsStack(true); 7408 7409 // And our return value (tls address) is in the standard call return value 7410 // location. 7411 unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; 7412 return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(), 7413 Chain.getValue(1)); 7414 } 7415 7416 if (Subtarget->isTargetWindows()) { 7417 // Just use the implicit TLS architecture 7418 // Need to generate someting similar to: 7419 // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage 7420 // ; from TEB 7421 // mov ecx, dword [rel _tls_index]: Load index (from C runtime) 7422 // mov rcx, qword [rdx+rcx*8] 7423 // mov eax, .tls$:tlsvar 7424 // [rax+rcx] contains the address 7425 // Windows 64bit: gs:0x58 7426 // Windows 32bit: fs:__tls_array 7427 7428 // If GV is an alias then use the aliasee for determining 7429 // thread-localness. 7430 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 7431 GV = GA->resolveAliasedGlobal(false); 7432 DebugLoc dl = GA->getDebugLoc(); 7433 SDValue Chain = DAG.getEntryNode(); 7434 7435 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or 7436 // %gs:0x58 (64-bit). 7437 Value *Ptr = Constant::getNullValue(Subtarget->is64Bit() 7438 ? Type::getInt8PtrTy(*DAG.getContext(), 7439 256) 7440 : Type::getInt32PtrTy(*DAG.getContext(), 7441 257)); 7442 7443 SDValue ThreadPointer = DAG.getLoad(getPointerTy(), dl, Chain, 7444 Subtarget->is64Bit() 7445 ? DAG.getIntPtrConstant(0x58) 7446 : DAG.getExternalSymbol("_tls_array", 7447 getPointerTy()), 7448 MachinePointerInfo(Ptr), 7449 false, false, false, 0); 7450 7451 // Load the _tls_index variable 7452 SDValue IDX = DAG.getExternalSymbol("_tls_index", getPointerTy()); 7453 if (Subtarget->is64Bit()) 7454 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, getPointerTy(), Chain, 7455 IDX, MachinePointerInfo(), MVT::i32, 7456 false, false, 0); 7457 else 7458 IDX = DAG.getLoad(getPointerTy(), dl, Chain, IDX, MachinePointerInfo(), 7459 false, false, false, 0); 7460 7461 SDValue Scale = DAG.getConstant(Log2_64_Ceil(TD->getPointerSize()), 7462 getPointerTy()); 7463 IDX = DAG.getNode(ISD::SHL, dl, getPointerTy(), IDX, Scale); 7464 7465 SDValue res = DAG.getNode(ISD::ADD, dl, getPointerTy(), ThreadPointer, IDX); 7466 res = DAG.getLoad(getPointerTy(), dl, Chain, res, MachinePointerInfo(), 7467 false, false, false, 0); 7468 7469 // Get the offset of start of .tls section 7470 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, 7471 GA->getValueType(0), 7472 GA->getOffset(), X86II::MO_SECREL); 7473 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), TGA); 7474 7475 // The address of the thread local variable is the add of the thread 7476 // pointer with the offset of the variable. 7477 return DAG.getNode(ISD::ADD, dl, getPointerTy(), res, Offset); 7478 } 7479 7480 llvm_unreachable("TLS not implemented for this target."); 7481} 7482 7483 7484/// LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values 7485/// and take a 2 x i32 value to shift plus a shift amount. 7486SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const{ 7487 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 7488 EVT VT = Op.getValueType(); 7489 unsigned VTBits = VT.getSizeInBits(); 7490 DebugLoc dl = Op.getDebugLoc(); 7491 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 7492 SDValue ShOpLo = Op.getOperand(0); 7493 SDValue ShOpHi = Op.getOperand(1); 7494 SDValue ShAmt = Op.getOperand(2); 7495 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, 7496 DAG.getConstant(VTBits - 1, MVT::i8)) 7497 : DAG.getConstant(0, VT); 7498 7499 SDValue Tmp2, Tmp3; 7500 if (Op.getOpcode() == ISD::SHL_PARTS) { 7501 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt); 7502 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 7503 } else { 7504 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt); 7505 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt); 7506 } 7507 7508 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, 7509 DAG.getConstant(VTBits, MVT::i8)); 7510 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, 7511 AndNode, DAG.getConstant(0, MVT::i8)); 7512 7513 SDValue Hi, Lo; 7514 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 7515 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 7516 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 7517 7518 if (Op.getOpcode() == ISD::SHL_PARTS) { 7519 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 7520 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 7521 } else { 7522 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 7523 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 7524 } 7525 7526 SDValue Ops[2] = { Lo, Hi }; 7527 return DAG.getMergeValues(Ops, 2, dl); 7528} 7529 7530SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, 7531 SelectionDAG &DAG) const { 7532 EVT SrcVT = Op.getOperand(0).getValueType(); 7533 7534 if (SrcVT.isVector()) 7535 return SDValue(); 7536 7537 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 7538 "Unknown SINT_TO_FP to lower!"); 7539 7540 // These are really Legal; return the operand so the caller accepts it as 7541 // Legal. 7542 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 7543 return Op; 7544 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && 7545 Subtarget->is64Bit()) { 7546 return Op; 7547 } 7548 7549 DebugLoc dl = Op.getDebugLoc(); 7550 unsigned Size = SrcVT.getSizeInBits()/8; 7551 MachineFunction &MF = DAG.getMachineFunction(); 7552 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false); 7553 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7554 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 7555 StackSlot, 7556 MachinePointerInfo::getFixedStack(SSFI), 7557 false, false, 0); 7558 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG); 7559} 7560 7561SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, 7562 SDValue StackSlot, 7563 SelectionDAG &DAG) const { 7564 // Build the FILD 7565 DebugLoc DL = Op.getDebugLoc(); 7566 SDVTList Tys; 7567 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 7568 if (useSSE) 7569 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); 7570 else 7571 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 7572 7573 unsigned ByteSize = SrcVT.getSizeInBits()/8; 7574 7575 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(StackSlot); 7576 MachineMemOperand *MMO; 7577 if (FI) { 7578 int SSFI = FI->getIndex(); 7579 MMO = 7580 DAG.getMachineFunction() 7581 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7582 MachineMemOperand::MOLoad, ByteSize, ByteSize); 7583 } else { 7584 MMO = cast<LoadSDNode>(StackSlot)->getMemOperand(); 7585 StackSlot = StackSlot.getOperand(1); 7586 } 7587 SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) }; 7588 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG : 7589 X86ISD::FILD, DL, 7590 Tys, Ops, array_lengthof(Ops), 7591 SrcVT, MMO); 7592 7593 if (useSSE) { 7594 Chain = Result.getValue(1); 7595 SDValue InFlag = Result.getValue(2); 7596 7597 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 7598 // shouldn't be necessary except that RFP cannot be live across 7599 // multiple blocks. When stackifier is fixed, they can be uncoupled. 7600 MachineFunction &MF = DAG.getMachineFunction(); 7601 unsigned SSFISize = Op.getValueType().getSizeInBits()/8; 7602 int SSFI = MF.getFrameInfo()->CreateStackObject(SSFISize, SSFISize, false); 7603 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7604 Tys = DAG.getVTList(MVT::Other); 7605 SDValue Ops[] = { 7606 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag 7607 }; 7608 MachineMemOperand *MMO = 7609 DAG.getMachineFunction() 7610 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7611 MachineMemOperand::MOStore, SSFISize, SSFISize); 7612 7613 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, 7614 Ops, array_lengthof(Ops), 7615 Op.getValueType(), MMO); 7616 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot, 7617 MachinePointerInfo::getFixedStack(SSFI), 7618 false, false, false, 0); 7619 } 7620 7621 return Result; 7622} 7623 7624// LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion. 7625SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op, 7626 SelectionDAG &DAG) const { 7627 // This algorithm is not obvious. Here it is what we're trying to output: 7628 /* 7629 movq %rax, %xmm0 7630 punpckldq (c0), %xmm0 // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U } 7631 subpd (c1), %xmm0 // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 } 7632 #ifdef __SSE3__ 7633 haddpd %xmm0, %xmm0 7634 #else 7635 pshufd $0x4e, %xmm0, %xmm1 7636 addpd %xmm1, %xmm0 7637 #endif 7638 */ 7639 7640 DebugLoc dl = Op.getDebugLoc(); 7641 LLVMContext *Context = DAG.getContext(); 7642 7643 // Build some magic constants. 7644 const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 }; 7645 Constant *C0 = ConstantDataVector::get(*Context, CV0); 7646 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16); 7647 7648 SmallVector<Constant*,2> CV1; 7649 CV1.push_back( 7650 ConstantFP::get(*Context, APFloat(APInt(64, 0x4330000000000000ULL)))); 7651 CV1.push_back( 7652 ConstantFP::get(*Context, APFloat(APInt(64, 0x4530000000000000ULL)))); 7653 Constant *C1 = ConstantVector::get(CV1); 7654 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16); 7655 7656 // Load the 64-bit value into an XMM register. 7657 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, 7658 Op.getOperand(0)); 7659 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, 7660 MachinePointerInfo::getConstantPool(), 7661 false, false, false, 16); 7662 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, 7663 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1), 7664 CLod0); 7665 7666 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, 7667 MachinePointerInfo::getConstantPool(), 7668 false, false, false, 16); 7669 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1); 7670 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); 7671 SDValue Result; 7672 7673 if (Subtarget->hasSSE3()) { 7674 // FIXME: The 'haddpd' instruction may be slower than 'movhlps + addsd'. 7675 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); 7676 } else { 7677 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub); 7678 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32, 7679 S2F, 0x4E, DAG); 7680 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, 7681 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle), 7682 Sub); 7683 } 7684 7685 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, 7686 DAG.getIntPtrConstant(0)); 7687} 7688 7689// LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion. 7690SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op, 7691 SelectionDAG &DAG) const { 7692 DebugLoc dl = Op.getDebugLoc(); 7693 // FP constant to bias correct the final result. 7694 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), 7695 MVT::f64); 7696 7697 // Load the 32-bit value into an XMM register. 7698 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 7699 Op.getOperand(0)); 7700 7701 // Zero out the upper parts of the register. 7702 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG); 7703 7704 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 7705 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), 7706 DAG.getIntPtrConstant(0)); 7707 7708 // Or the load with the bias. 7709 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, 7710 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 7711 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 7712 MVT::v2f64, Load)), 7713 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 7714 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 7715 MVT::v2f64, Bias))); 7716 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 7717 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), 7718 DAG.getIntPtrConstant(0)); 7719 7720 // Subtract the bias. 7721 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); 7722 7723 // Handle final rounding. 7724 EVT DestVT = Op.getValueType(); 7725 7726 if (DestVT.bitsLT(MVT::f64)) 7727 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 7728 DAG.getIntPtrConstant(0)); 7729 if (DestVT.bitsGT(MVT::f64)) 7730 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 7731 7732 // Handle final rounding. 7733 return Sub; 7734} 7735 7736SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, 7737 SelectionDAG &DAG) const { 7738 SDValue N0 = Op.getOperand(0); 7739 DebugLoc dl = Op.getDebugLoc(); 7740 7741 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't 7742 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform 7743 // the optimization here. 7744 if (DAG.SignBitIsZero(N0)) 7745 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); 7746 7747 EVT SrcVT = N0.getValueType(); 7748 EVT DstVT = Op.getValueType(); 7749 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) 7750 return LowerUINT_TO_FP_i64(Op, DAG); 7751 if (SrcVT == MVT::i32 && X86ScalarSSEf64) 7752 return LowerUINT_TO_FP_i32(Op, DAG); 7753 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) 7754 return SDValue(); 7755 7756 // Make a 64-bit buffer, and use it to build an FILD. 7757 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); 7758 if (SrcVT == MVT::i32) { 7759 SDValue WordOff = DAG.getConstant(4, getPointerTy()); 7760 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, 7761 getPointerTy(), StackSlot, WordOff); 7762 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 7763 StackSlot, MachinePointerInfo(), 7764 false, false, 0); 7765 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), 7766 OffsetSlot, MachinePointerInfo(), 7767 false, false, 0); 7768 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); 7769 return Fild; 7770 } 7771 7772 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); 7773 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 7774 StackSlot, MachinePointerInfo(), 7775 false, false, 0); 7776 // For i64 source, we need to add the appropriate power of 2 if the input 7777 // was negative. This is the same as the optimization in 7778 // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here, 7779 // we must be careful to do the computation in x87 extended precision, not 7780 // in SSE. (The generic code can't know it's OK to do this, or how to.) 7781 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 7782 MachineMemOperand *MMO = 7783 DAG.getMachineFunction() 7784 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7785 MachineMemOperand::MOLoad, 8, 8); 7786 7787 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); 7788 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; 7789 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, 3, 7790 MVT::i64, MMO); 7791 7792 APInt FF(32, 0x5F800000ULL); 7793 7794 // Check whether the sign bit is set. 7795 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 7796 Op.getOperand(0), DAG.getConstant(0, MVT::i64), 7797 ISD::SETLT); 7798 7799 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 7800 SDValue FudgePtr = DAG.getConstantPool( 7801 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 7802 getPointerTy()); 7803 7804 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 7805 SDValue Zero = DAG.getIntPtrConstant(0); 7806 SDValue Four = DAG.getIntPtrConstant(4); 7807 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 7808 Zero, Four); 7809 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset); 7810 7811 // Load the value out, extending it from f32 to f80. 7812 // FIXME: Avoid the extend by constructing the right constant pool? 7813 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), 7814 FudgePtr, MachinePointerInfo::getConstantPool(), 7815 MVT::f32, false, false, 4); 7816 // Extend everything to 80 bits to force it to be done on x87. 7817 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); 7818 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0)); 7819} 7820 7821std::pair<SDValue,SDValue> X86TargetLowering:: 7822FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned, bool IsReplace) const { 7823 DebugLoc DL = Op.getDebugLoc(); 7824 7825 EVT DstTy = Op.getValueType(); 7826 7827 if (!IsSigned && !isIntegerTypeFTOL(DstTy)) { 7828 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); 7829 DstTy = MVT::i64; 7830 } 7831 7832 assert(DstTy.getSimpleVT() <= MVT::i64 && 7833 DstTy.getSimpleVT() >= MVT::i16 && 7834 "Unknown FP_TO_INT to lower!"); 7835 7836 // These are really Legal. 7837 if (DstTy == MVT::i32 && 7838 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 7839 return std::make_pair(SDValue(), SDValue()); 7840 if (Subtarget->is64Bit() && 7841 DstTy == MVT::i64 && 7842 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 7843 return std::make_pair(SDValue(), SDValue()); 7844 7845 // We lower FP->int64 either into FISTP64 followed by a load from a temporary 7846 // stack slot, or into the FTOL runtime function. 7847 MachineFunction &MF = DAG.getMachineFunction(); 7848 unsigned MemSize = DstTy.getSizeInBits()/8; 7849 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false); 7850 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7851 7852 unsigned Opc; 7853 if (!IsSigned && isIntegerTypeFTOL(DstTy)) 7854 Opc = X86ISD::WIN_FTOL; 7855 else 7856 switch (DstTy.getSimpleVT().SimpleTy) { 7857 default: llvm_unreachable("Invalid FP_TO_SINT to lower!"); 7858 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 7859 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 7860 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 7861 } 7862 7863 SDValue Chain = DAG.getEntryNode(); 7864 SDValue Value = Op.getOperand(0); 7865 EVT TheVT = Op.getOperand(0).getValueType(); 7866 // FIXME This causes a redundant load/store if the SSE-class value is already 7867 // in memory, such as if it is on the callstack. 7868 if (isScalarFPTypeInSSEReg(TheVT)) { 7869 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 7870 Chain = DAG.getStore(Chain, DL, Value, StackSlot, 7871 MachinePointerInfo::getFixedStack(SSFI), 7872 false, false, 0); 7873 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 7874 SDValue Ops[] = { 7875 Chain, StackSlot, DAG.getValueType(TheVT) 7876 }; 7877 7878 MachineMemOperand *MMO = 7879 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7880 MachineMemOperand::MOLoad, MemSize, MemSize); 7881 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, 3, 7882 DstTy, MMO); 7883 Chain = Value.getValue(1); 7884 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false); 7885 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 7886 } 7887 7888 MachineMemOperand *MMO = 7889 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 7890 MachineMemOperand::MOStore, MemSize, MemSize); 7891 7892 if (Opc != X86ISD::WIN_FTOL) { 7893 // Build the FP_TO_INT*_IN_MEM 7894 SDValue Ops[] = { Chain, Value, StackSlot }; 7895 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), 7896 Ops, 3, DstTy, MMO); 7897 return std::make_pair(FIST, StackSlot); 7898 } else { 7899 SDValue ftol = DAG.getNode(X86ISD::WIN_FTOL, DL, 7900 DAG.getVTList(MVT::Other, MVT::Glue), 7901 Chain, Value); 7902 SDValue eax = DAG.getCopyFromReg(ftol, DL, X86::EAX, 7903 MVT::i32, ftol.getValue(1)); 7904 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), DL, X86::EDX, 7905 MVT::i32, eax.getValue(2)); 7906 SDValue Ops[] = { eax, edx }; 7907 SDValue pair = IsReplace 7908 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops, 2) 7909 : DAG.getMergeValues(Ops, 2, DL); 7910 return std::make_pair(pair, SDValue()); 7911 } 7912} 7913 7914SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, 7915 SelectionDAG &DAG) const { 7916 if (Op.getValueType().isVector()) 7917 return SDValue(); 7918 7919 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, 7920 /*IsSigned=*/ true, /*IsReplace=*/ false); 7921 SDValue FIST = Vals.first, StackSlot = Vals.second; 7922 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal. 7923 if (FIST.getNode() == 0) return Op; 7924 7925 if (StackSlot.getNode()) 7926 // Load the result. 7927 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 7928 FIST, StackSlot, MachinePointerInfo(), 7929 false, false, false, 0); 7930 7931 // The node is the result. 7932 return FIST; 7933} 7934 7935SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op, 7936 SelectionDAG &DAG) const { 7937 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, 7938 /*IsSigned=*/ false, /*IsReplace=*/ false); 7939 SDValue FIST = Vals.first, StackSlot = Vals.second; 7940 assert(FIST.getNode() && "Unexpected failure"); 7941 7942 if (StackSlot.getNode()) 7943 // Load the result. 7944 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 7945 FIST, StackSlot, MachinePointerInfo(), 7946 false, false, false, 0); 7947 7948 // The node is the result. 7949 return FIST; 7950} 7951 7952SDValue X86TargetLowering::LowerFABS(SDValue Op, 7953 SelectionDAG &DAG) const { 7954 LLVMContext *Context = DAG.getContext(); 7955 DebugLoc dl = Op.getDebugLoc(); 7956 EVT VT = Op.getValueType(); 7957 EVT EltVT = VT; 7958 if (VT.isVector()) 7959 EltVT = VT.getVectorElementType(); 7960 Constant *C; 7961 if (EltVT == MVT::f64) { 7962 C = ConstantVector::getSplat(2, 7963 ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))))); 7964 } else { 7965 C = ConstantVector::getSplat(4, 7966 ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))))); 7967 } 7968 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7969 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 7970 MachinePointerInfo::getConstantPool(), 7971 false, false, false, 16); 7972 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask); 7973} 7974 7975SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { 7976 LLVMContext *Context = DAG.getContext(); 7977 DebugLoc dl = Op.getDebugLoc(); 7978 EVT VT = Op.getValueType(); 7979 EVT EltVT = VT; 7980 unsigned NumElts = VT == MVT::f64 ? 2 : 4; 7981 if (VT.isVector()) { 7982 EltVT = VT.getVectorElementType(); 7983 NumElts = VT.getVectorNumElements(); 7984 } 7985 Constant *C; 7986 if (EltVT == MVT::f64) 7987 C = ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63))); 7988 else 7989 C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31))); 7990 C = ConstantVector::getSplat(NumElts, C); 7991 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 7992 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 7993 MachinePointerInfo::getConstantPool(), 7994 false, false, false, 16); 7995 if (VT.isVector()) { 7996 MVT XORVT = VT.getSizeInBits() == 128 ? MVT::v2i64 : MVT::v4i64; 7997 return DAG.getNode(ISD::BITCAST, dl, VT, 7998 DAG.getNode(ISD::XOR, dl, XORVT, 7999 DAG.getNode(ISD::BITCAST, dl, XORVT, 8000 Op.getOperand(0)), 8001 DAG.getNode(ISD::BITCAST, dl, XORVT, Mask))); 8002 } 8003 8004 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask); 8005} 8006 8007SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { 8008 LLVMContext *Context = DAG.getContext(); 8009 SDValue Op0 = Op.getOperand(0); 8010 SDValue Op1 = Op.getOperand(1); 8011 DebugLoc dl = Op.getDebugLoc(); 8012 EVT VT = Op.getValueType(); 8013 EVT SrcVT = Op1.getValueType(); 8014 8015 // If second operand is smaller, extend it first. 8016 if (SrcVT.bitsLT(VT)) { 8017 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1); 8018 SrcVT = VT; 8019 } 8020 // And if it is bigger, shrink it first. 8021 if (SrcVT.bitsGT(VT)) { 8022 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1)); 8023 SrcVT = VT; 8024 } 8025 8026 // At this point the operands and the result should have the same 8027 // type, and that won't be f80 since that is not custom lowered. 8028 8029 // First get the sign bit of second operand. 8030 SmallVector<Constant*,4> CV; 8031 if (SrcVT == MVT::f64) { 8032 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63)))); 8033 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 8034 } else { 8035 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)))); 8036 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8037 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8038 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8039 } 8040 Constant *C = ConstantVector::get(CV); 8041 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 8042 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx, 8043 MachinePointerInfo::getConstantPool(), 8044 false, false, false, 16); 8045 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1); 8046 8047 // Shift sign bit right or left if the two operands have different types. 8048 if (SrcVT.bitsGT(VT)) { 8049 // Op0 is MVT::f32, Op1 is MVT::f64. 8050 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit); 8051 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit, 8052 DAG.getConstant(32, MVT::i32)); 8053 SignBit = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, SignBit); 8054 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit, 8055 DAG.getIntPtrConstant(0)); 8056 } 8057 8058 // Clear first operand sign bit. 8059 CV.clear(); 8060 if (VT == MVT::f64) { 8061 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))))); 8062 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 8063 } else { 8064 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))))); 8065 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8066 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8067 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 8068 } 8069 C = ConstantVector::get(CV); 8070 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 8071 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 8072 MachinePointerInfo::getConstantPool(), 8073 false, false, false, 16); 8074 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2); 8075 8076 // Or the value with the sign bit. 8077 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit); 8078} 8079 8080SDValue X86TargetLowering::LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const { 8081 SDValue N0 = Op.getOperand(0); 8082 DebugLoc dl = Op.getDebugLoc(); 8083 EVT VT = Op.getValueType(); 8084 8085 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1). 8086 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0, 8087 DAG.getConstant(1, VT)); 8088 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT)); 8089} 8090 8091/// Emit nodes that will be selected as "test Op0,Op0", or something 8092/// equivalent. 8093SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, 8094 SelectionDAG &DAG) const { 8095 DebugLoc dl = Op.getDebugLoc(); 8096 8097 // CF and OF aren't always set the way we want. Determine which 8098 // of these we need. 8099 bool NeedCF = false; 8100 bool NeedOF = false; 8101 switch (X86CC) { 8102 default: break; 8103 case X86::COND_A: case X86::COND_AE: 8104 case X86::COND_B: case X86::COND_BE: 8105 NeedCF = true; 8106 break; 8107 case X86::COND_G: case X86::COND_GE: 8108 case X86::COND_L: case X86::COND_LE: 8109 case X86::COND_O: case X86::COND_NO: 8110 NeedOF = true; 8111 break; 8112 } 8113 8114 // See if we can use the EFLAGS value from the operand instead of 8115 // doing a separate TEST. TEST always sets OF and CF to 0, so unless 8116 // we prove that the arithmetic won't overflow, we can't use OF or CF. 8117 if (Op.getResNo() != 0 || NeedOF || NeedCF) 8118 // Emit a CMP with 0, which is the TEST pattern. 8119 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, 8120 DAG.getConstant(0, Op.getValueType())); 8121 8122 unsigned Opcode = 0; 8123 unsigned NumOperands = 0; 8124 switch (Op.getNode()->getOpcode()) { 8125 case ISD::ADD: 8126 // Due to an isel shortcoming, be conservative if this add is likely to be 8127 // selected as part of a load-modify-store instruction. When the root node 8128 // in a match is a store, isel doesn't know how to remap non-chain non-flag 8129 // uses of other nodes in the match, such as the ADD in this case. This 8130 // leads to the ADD being left around and reselected, with the result being 8131 // two adds in the output. Alas, even if none our users are stores, that 8132 // doesn't prove we're O.K. Ergo, if we have any parents that aren't 8133 // CopyToReg or SETCC, eschew INC/DEC. A better fix seems to require 8134 // climbing the DAG back to the root, and it doesn't seem to be worth the 8135 // effort. 8136 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 8137 UE = Op.getNode()->use_end(); UI != UE; ++UI) 8138 if (UI->getOpcode() != ISD::CopyToReg && 8139 UI->getOpcode() != ISD::SETCC && 8140 UI->getOpcode() != ISD::STORE) 8141 goto default_case; 8142 8143 if (ConstantSDNode *C = 8144 dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) { 8145 // An add of one will be selected as an INC. 8146 if (C->getAPIntValue() == 1) { 8147 Opcode = X86ISD::INC; 8148 NumOperands = 1; 8149 break; 8150 } 8151 8152 // An add of negative one (subtract of one) will be selected as a DEC. 8153 if (C->getAPIntValue().isAllOnesValue()) { 8154 Opcode = X86ISD::DEC; 8155 NumOperands = 1; 8156 break; 8157 } 8158 } 8159 8160 // Otherwise use a regular EFLAGS-setting add. 8161 Opcode = X86ISD::ADD; 8162 NumOperands = 2; 8163 break; 8164 case ISD::AND: { 8165 // If the primary and result isn't used, don't bother using X86ISD::AND, 8166 // because a TEST instruction will be better. 8167 bool NonFlagUse = false; 8168 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 8169 UE = Op.getNode()->use_end(); UI != UE; ++UI) { 8170 SDNode *User = *UI; 8171 unsigned UOpNo = UI.getOperandNo(); 8172 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) { 8173 // Look pass truncate. 8174 UOpNo = User->use_begin().getOperandNo(); 8175 User = *User->use_begin(); 8176 } 8177 8178 if (User->getOpcode() != ISD::BRCOND && 8179 User->getOpcode() != ISD::SETCC && 8180 (User->getOpcode() != ISD::SELECT || UOpNo != 0)) { 8181 NonFlagUse = true; 8182 break; 8183 } 8184 } 8185 8186 if (!NonFlagUse) 8187 break; 8188 } 8189 // FALL THROUGH 8190 case ISD::SUB: 8191 case ISD::OR: 8192 case ISD::XOR: 8193 // Due to the ISEL shortcoming noted above, be conservative if this op is 8194 // likely to be selected as part of a load-modify-store instruction. 8195 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 8196 UE = Op.getNode()->use_end(); UI != UE; ++UI) 8197 if (UI->getOpcode() == ISD::STORE) 8198 goto default_case; 8199 8200 // Otherwise use a regular EFLAGS-setting instruction. 8201 switch (Op.getNode()->getOpcode()) { 8202 default: llvm_unreachable("unexpected operator!"); 8203 case ISD::SUB: Opcode = X86ISD::SUB; break; 8204 case ISD::OR: Opcode = X86ISD::OR; break; 8205 case ISD::XOR: Opcode = X86ISD::XOR; break; 8206 case ISD::AND: Opcode = X86ISD::AND; break; 8207 } 8208 8209 NumOperands = 2; 8210 break; 8211 case X86ISD::ADD: 8212 case X86ISD::SUB: 8213 case X86ISD::INC: 8214 case X86ISD::DEC: 8215 case X86ISD::OR: 8216 case X86ISD::XOR: 8217 case X86ISD::AND: 8218 return SDValue(Op.getNode(), 1); 8219 default: 8220 default_case: 8221 break; 8222 } 8223 8224 if (Opcode == 0) 8225 // Emit a CMP with 0, which is the TEST pattern. 8226 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, 8227 DAG.getConstant(0, Op.getValueType())); 8228 8229 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 8230 SmallVector<SDValue, 4> Ops; 8231 for (unsigned i = 0; i != NumOperands; ++i) 8232 Ops.push_back(Op.getOperand(i)); 8233 8234 SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands); 8235 DAG.ReplaceAllUsesWith(Op, New); 8236 return SDValue(New.getNode(), 1); 8237} 8238 8239/// Emit nodes that will be selected as "cmp Op0,Op1", or something 8240/// equivalent. 8241SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, 8242 SelectionDAG &DAG) const { 8243 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) 8244 if (C->getAPIntValue() == 0) 8245 return EmitTest(Op0, X86CC, DAG); 8246 8247 DebugLoc dl = Op0.getDebugLoc(); 8248 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); 8249} 8250 8251/// Convert a comparison if required by the subtarget. 8252SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp, 8253 SelectionDAG &DAG) const { 8254 // If the subtarget does not support the FUCOMI instruction, floating-point 8255 // comparisons have to be converted. 8256 if (Subtarget->hasCMov() || 8257 Cmp.getOpcode() != X86ISD::CMP || 8258 !Cmp.getOperand(0).getValueType().isFloatingPoint() || 8259 !Cmp.getOperand(1).getValueType().isFloatingPoint()) 8260 return Cmp; 8261 8262 // The instruction selector will select an FUCOM instruction instead of 8263 // FUCOMI, which writes the comparison result to FPSW instead of EFLAGS. Hence 8264 // build an SDNode sequence that transfers the result from FPSW into EFLAGS: 8265 // (X86sahf (trunc (srl (X86fp_stsw (trunc (X86cmp ...)), 8)))) 8266 DebugLoc dl = Cmp.getDebugLoc(); 8267 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); 8268 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW); 8269 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, 8270 DAG.getConstant(8, MVT::i8)); 8271 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); 8272 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl); 8273} 8274 8275/// LowerToBT - Result of 'and' is compared against zero. Turn it into a BT node 8276/// if it's possible. 8277SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC, 8278 DebugLoc dl, SelectionDAG &DAG) const { 8279 SDValue Op0 = And.getOperand(0); 8280 SDValue Op1 = And.getOperand(1); 8281 if (Op0.getOpcode() == ISD::TRUNCATE) 8282 Op0 = Op0.getOperand(0); 8283 if (Op1.getOpcode() == ISD::TRUNCATE) 8284 Op1 = Op1.getOperand(0); 8285 8286 SDValue LHS, RHS; 8287 if (Op1.getOpcode() == ISD::SHL) 8288 std::swap(Op0, Op1); 8289 if (Op0.getOpcode() == ISD::SHL) { 8290 if (ConstantSDNode *And00C = dyn_cast<ConstantSDNode>(Op0.getOperand(0))) 8291 if (And00C->getZExtValue() == 1) { 8292 // If we looked past a truncate, check that it's only truncating away 8293 // known zeros. 8294 unsigned BitWidth = Op0.getValueSizeInBits(); 8295 unsigned AndBitWidth = And.getValueSizeInBits(); 8296 if (BitWidth > AndBitWidth) { 8297 APInt Zeros, Ones; 8298 DAG.ComputeMaskedBits(Op0, Zeros, Ones); 8299 if (Zeros.countLeadingOnes() < BitWidth - AndBitWidth) 8300 return SDValue(); 8301 } 8302 LHS = Op1; 8303 RHS = Op0.getOperand(1); 8304 } 8305 } else if (Op1.getOpcode() == ISD::Constant) { 8306 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1); 8307 uint64_t AndRHSVal = AndRHS->getZExtValue(); 8308 SDValue AndLHS = Op0; 8309 8310 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) { 8311 LHS = AndLHS.getOperand(0); 8312 RHS = AndLHS.getOperand(1); 8313 } 8314 8315 // Use BT if the immediate can't be encoded in a TEST instruction. 8316 if (!isUInt<32>(AndRHSVal) && isPowerOf2_64(AndRHSVal)) { 8317 LHS = AndLHS; 8318 RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), LHS.getValueType()); 8319 } 8320 } 8321 8322 if (LHS.getNode()) { 8323 // If LHS is i8, promote it to i32 with any_extend. There is no i8 BT 8324 // instruction. Since the shift amount is in-range-or-undefined, we know 8325 // that doing a bittest on the i32 value is ok. We extend to i32 because 8326 // the encoding for the i16 version is larger than the i32 version. 8327 // Also promote i16 to i32 for performance / code size reason. 8328 if (LHS.getValueType() == MVT::i8 || 8329 LHS.getValueType() == MVT::i16) 8330 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); 8331 8332 // If the operand types disagree, extend the shift amount to match. Since 8333 // BT ignores high bits (like shifts) we can use anyextend. 8334 if (LHS.getValueType() != RHS.getValueType()) 8335 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS); 8336 8337 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); 8338 unsigned Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B; 8339 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 8340 DAG.getConstant(Cond, MVT::i8), BT); 8341 } 8342 8343 return SDValue(); 8344} 8345 8346SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 8347 8348 if (Op.getValueType().isVector()) return LowerVSETCC(Op, DAG); 8349 8350 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 8351 SDValue Op0 = Op.getOperand(0); 8352 SDValue Op1 = Op.getOperand(1); 8353 DebugLoc dl = Op.getDebugLoc(); 8354 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 8355 8356 // Optimize to BT if possible. 8357 // Lower (X & (1 << N)) == 0 to BT(X, N). 8358 // Lower ((X >>u N) & 1) != 0 to BT(X, N). 8359 // Lower ((X >>s N) & 1) != 0 to BT(X, N). 8360 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && 8361 Op1.getOpcode() == ISD::Constant && 8362 cast<ConstantSDNode>(Op1)->isNullValue() && 8363 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 8364 SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG); 8365 if (NewSetCC.getNode()) 8366 return NewSetCC; 8367 } 8368 8369 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms of 8370 // these. 8371 if (Op1.getOpcode() == ISD::Constant && 8372 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 || 8373 cast<ConstantSDNode>(Op1)->isNullValue()) && 8374 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 8375 8376 // If the input is a setcc, then reuse the input setcc or use a new one with 8377 // the inverted condition. 8378 if (Op0.getOpcode() == X86ISD::SETCC) { 8379 X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0); 8380 bool Invert = (CC == ISD::SETNE) ^ 8381 cast<ConstantSDNode>(Op1)->isNullValue(); 8382 if (!Invert) return Op0; 8383 8384 CCode = X86::GetOppositeBranchCondition(CCode); 8385 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 8386 DAG.getConstant(CCode, MVT::i8), Op0.getOperand(1)); 8387 } 8388 } 8389 8390 bool isFP = Op1.getValueType().isFloatingPoint(); 8391 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); 8392 if (X86CC == X86::COND_INVALID) 8393 return SDValue(); 8394 8395 SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, DAG); 8396 EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG); 8397 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 8398 DAG.getConstant(X86CC, MVT::i8), EFLAGS); 8399} 8400 8401// Lower256IntVSETCC - Break a VSETCC 256-bit integer VSETCC into two new 128 8402// ones, and then concatenate the result back. 8403static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) { 8404 EVT VT = Op.getValueType(); 8405 8406 assert(VT.getSizeInBits() == 256 && Op.getOpcode() == ISD::SETCC && 8407 "Unsupported value type for operation"); 8408 8409 unsigned NumElems = VT.getVectorNumElements(); 8410 DebugLoc dl = Op.getDebugLoc(); 8411 SDValue CC = Op.getOperand(2); 8412 8413 // Extract the LHS vectors 8414 SDValue LHS = Op.getOperand(0); 8415 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl); 8416 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl); 8417 8418 // Extract the RHS vectors 8419 SDValue RHS = Op.getOperand(1); 8420 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl); 8421 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl); 8422 8423 // Issue the operation on the smaller types and concatenate the result back 8424 MVT EltVT = VT.getVectorElementType().getSimpleVT(); 8425 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); 8426 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, 8427 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC), 8428 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC)); 8429} 8430 8431 8432SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const { 8433 SDValue Cond; 8434 SDValue Op0 = Op.getOperand(0); 8435 SDValue Op1 = Op.getOperand(1); 8436 SDValue CC = Op.getOperand(2); 8437 EVT VT = Op.getValueType(); 8438 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 8439 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 8440 DebugLoc dl = Op.getDebugLoc(); 8441 8442 if (isFP) { 8443 unsigned SSECC = 8; 8444 EVT EltVT = Op0.getValueType().getVectorElementType(); 8445 assert(EltVT == MVT::f32 || EltVT == MVT::f64); (void)EltVT; 8446 8447 bool Swap = false; 8448 8449 // SSE Condition code mapping: 8450 // 0 - EQ 8451 // 1 - LT 8452 // 2 - LE 8453 // 3 - UNORD 8454 // 4 - NEQ 8455 // 5 - NLT 8456 // 6 - NLE 8457 // 7 - ORD 8458 switch (SetCCOpcode) { 8459 default: break; 8460 case ISD::SETOEQ: 8461 case ISD::SETEQ: SSECC = 0; break; 8462 case ISD::SETOGT: 8463 case ISD::SETGT: Swap = true; // Fallthrough 8464 case ISD::SETLT: 8465 case ISD::SETOLT: SSECC = 1; break; 8466 case ISD::SETOGE: 8467 case ISD::SETGE: Swap = true; // Fallthrough 8468 case ISD::SETLE: 8469 case ISD::SETOLE: SSECC = 2; break; 8470 case ISD::SETUO: SSECC = 3; break; 8471 case ISD::SETUNE: 8472 case ISD::SETNE: SSECC = 4; break; 8473 case ISD::SETULE: Swap = true; 8474 case ISD::SETUGE: SSECC = 5; break; 8475 case ISD::SETULT: Swap = true; 8476 case ISD::SETUGT: SSECC = 6; break; 8477 case ISD::SETO: SSECC = 7; break; 8478 } 8479 if (Swap) 8480 std::swap(Op0, Op1); 8481 8482 // In the two special cases we can't handle, emit two comparisons. 8483 if (SSECC == 8) { 8484 if (SetCCOpcode == ISD::SETUEQ) { 8485 SDValue UNORD, EQ; 8486 UNORD = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1, 8487 DAG.getConstant(3, MVT::i8)); 8488 EQ = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1, 8489 DAG.getConstant(0, MVT::i8)); 8490 return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ); 8491 } 8492 if (SetCCOpcode == ISD::SETONE) { 8493 SDValue ORD, NEQ; 8494 ORD = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1, 8495 DAG.getConstant(7, MVT::i8)); 8496 NEQ = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1, 8497 DAG.getConstant(4, MVT::i8)); 8498 return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ); 8499 } 8500 llvm_unreachable("Illegal FP comparison"); 8501 } 8502 // Handle all other FP comparisons here. 8503 return DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1, 8504 DAG.getConstant(SSECC, MVT::i8)); 8505 } 8506 8507 // Break 256-bit integer vector compare into smaller ones. 8508 if (VT.getSizeInBits() == 256 && !Subtarget->hasAVX2()) 8509 return Lower256IntVSETCC(Op, DAG); 8510 8511 // We are handling one of the integer comparisons here. Since SSE only has 8512 // GT and EQ comparisons for integer, swapping operands and multiple 8513 // operations may be required for some comparisons. 8514 unsigned Opc = 0; 8515 bool Swap = false, Invert = false, FlipSigns = false; 8516 8517 switch (SetCCOpcode) { 8518 default: break; 8519 case ISD::SETNE: Invert = true; 8520 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break; 8521 case ISD::SETLT: Swap = true; 8522 case ISD::SETGT: Opc = X86ISD::PCMPGT; break; 8523 case ISD::SETGE: Swap = true; 8524 case ISD::SETLE: Opc = X86ISD::PCMPGT; Invert = true; break; 8525 case ISD::SETULT: Swap = true; 8526 case ISD::SETUGT: Opc = X86ISD::PCMPGT; FlipSigns = true; break; 8527 case ISD::SETUGE: Swap = true; 8528 case ISD::SETULE: Opc = X86ISD::PCMPGT; FlipSigns = true; Invert = true; break; 8529 } 8530 if (Swap) 8531 std::swap(Op0, Op1); 8532 8533 // Check that the operation in question is available (most are plain SSE2, 8534 // but PCMPGTQ and PCMPEQQ have different requirements). 8535 if (Opc == X86ISD::PCMPGT && VT == MVT::v2i64 && !Subtarget->hasSSE42()) 8536 return SDValue(); 8537 if (Opc == X86ISD::PCMPEQ && VT == MVT::v2i64 && !Subtarget->hasSSE41()) 8538 return SDValue(); 8539 8540 // Since SSE has no unsigned integer comparisons, we need to flip the sign 8541 // bits of the inputs before performing those operations. 8542 if (FlipSigns) { 8543 EVT EltVT = VT.getVectorElementType(); 8544 SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), 8545 EltVT); 8546 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 8547 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0], 8548 SignBits.size()); 8549 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec); 8550 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec); 8551 } 8552 8553 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); 8554 8555 // If the logical-not of the result is required, perform that now. 8556 if (Invert) 8557 Result = DAG.getNOT(dl, Result, VT); 8558 8559 return Result; 8560} 8561 8562// isX86LogicalCmp - Return true if opcode is a X86 logical comparison. 8563static bool isX86LogicalCmp(SDValue Op) { 8564 unsigned Opc = Op.getNode()->getOpcode(); 8565 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI || 8566 Opc == X86ISD::SAHF) 8567 return true; 8568 if (Op.getResNo() == 1 && 8569 (Opc == X86ISD::ADD || 8570 Opc == X86ISD::SUB || 8571 Opc == X86ISD::ADC || 8572 Opc == X86ISD::SBB || 8573 Opc == X86ISD::SMUL || 8574 Opc == X86ISD::UMUL || 8575 Opc == X86ISD::INC || 8576 Opc == X86ISD::DEC || 8577 Opc == X86ISD::OR || 8578 Opc == X86ISD::XOR || 8579 Opc == X86ISD::AND)) 8580 return true; 8581 8582 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL) 8583 return true; 8584 8585 return false; 8586} 8587 8588static bool isZero(SDValue V) { 8589 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V); 8590 return C && C->isNullValue(); 8591} 8592 8593static bool isAllOnes(SDValue V) { 8594 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V); 8595 return C && C->isAllOnesValue(); 8596} 8597 8598SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { 8599 bool addTest = true; 8600 SDValue Cond = Op.getOperand(0); 8601 SDValue Op1 = Op.getOperand(1); 8602 SDValue Op2 = Op.getOperand(2); 8603 DebugLoc DL = Op.getDebugLoc(); 8604 SDValue CC; 8605 8606 if (Cond.getOpcode() == ISD::SETCC) { 8607 SDValue NewCond = LowerSETCC(Cond, DAG); 8608 if (NewCond.getNode()) 8609 Cond = NewCond; 8610 } 8611 8612 // Handle the following cases related to max and min: 8613 // (a > b) ? (a-b) : 0 8614 // (a >= b) ? (a-b) : 0 8615 // (b < a) ? (a-b) : 0 8616 // (b <= a) ? (a-b) : 0 8617 // Comparison is removed to use EFLAGS from SUB. 8618 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2)) 8619 if (Cond.getOpcode() == X86ISD::SETCC && 8620 Cond.getOperand(1).getOpcode() == X86ISD::CMP && 8621 (Op1.getOpcode() == ISD::SUB || Op1.getOpcode() == X86ISD::SUB) && 8622 C->getAPIntValue() == 0) { 8623 SDValue Cmp = Cond.getOperand(1); 8624 unsigned CC = cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue(); 8625 if ((DAG.isEqualTo(Op1.getOperand(0), Cmp.getOperand(0)) && 8626 DAG.isEqualTo(Op1.getOperand(1), Cmp.getOperand(1)) && 8627 (CC == X86::COND_G || CC == X86::COND_GE || 8628 CC == X86::COND_A || CC == X86::COND_AE)) || 8629 (DAG.isEqualTo(Op1.getOperand(0), Cmp.getOperand(1)) && 8630 DAG.isEqualTo(Op1.getOperand(1), Cmp.getOperand(0)) && 8631 (CC == X86::COND_L || CC == X86::COND_LE || 8632 CC == X86::COND_B || CC == X86::COND_BE))) { 8633 8634 if (Op1.getOpcode() == ISD::SUB) { 8635 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i32); 8636 SDValue New = DAG.getNode(X86ISD::SUB, DL, VTs, 8637 Op1.getOperand(0), Op1.getOperand(1)); 8638 DAG.ReplaceAllUsesWith(Op1, New); 8639 Op1 = New; 8640 } 8641 8642 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 8643 unsigned NewCC = (CC == X86::COND_G || CC == X86::COND_GE || 8644 CC == X86::COND_L || 8645 CC == X86::COND_LE) ? X86::COND_GE : X86::COND_AE; 8646 SDValue Ops[] = { Op2, Op1, DAG.getConstant(NewCC, MVT::i8), 8647 SDValue(Op1.getNode(), 1) }; 8648 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops, array_lengthof(Ops)); 8649 } 8650 } 8651 8652 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y 8653 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y 8654 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y 8655 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y 8656 if (Cond.getOpcode() == X86ISD::SETCC && 8657 Cond.getOperand(1).getOpcode() == X86ISD::CMP && 8658 isZero(Cond.getOperand(1).getOperand(1))) { 8659 SDValue Cmp = Cond.getOperand(1); 8660 8661 unsigned CondCode =cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue(); 8662 8663 if ((isAllOnes(Op1) || isAllOnes(Op2)) && 8664 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) { 8665 SDValue Y = isAllOnes(Op2) ? Op1 : Op2; 8666 8667 SDValue CmpOp0 = Cmp.getOperand(0); 8668 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, 8669 CmpOp0, DAG.getConstant(1, CmpOp0.getValueType())); 8670 Cmp = ConvertCmpIfNecessary(Cmp, DAG); 8671 8672 SDValue Res = // Res = 0 or -1. 8673 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), 8674 DAG.getConstant(X86::COND_B, MVT::i8), Cmp); 8675 8676 if (isAllOnes(Op1) != (CondCode == X86::COND_E)) 8677 Res = DAG.getNOT(DL, Res, Res.getValueType()); 8678 8679 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Op2); 8680 if (N2C == 0 || !N2C->isNullValue()) 8681 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y); 8682 return Res; 8683 } 8684 } 8685 8686 // Look past (and (setcc_carry (cmp ...)), 1). 8687 if (Cond.getOpcode() == ISD::AND && 8688 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) { 8689 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 8690 if (C && C->getAPIntValue() == 1) 8691 Cond = Cond.getOperand(0); 8692 } 8693 8694 // If condition flag is set by a X86ISD::CMP, then use it as the condition 8695 // setting operand in place of the X86ISD::SETCC. 8696 unsigned CondOpcode = Cond.getOpcode(); 8697 if (CondOpcode == X86ISD::SETCC || 8698 CondOpcode == X86ISD::SETCC_CARRY) { 8699 CC = Cond.getOperand(0); 8700 8701 SDValue Cmp = Cond.getOperand(1); 8702 unsigned Opc = Cmp.getOpcode(); 8703 EVT VT = Op.getValueType(); 8704 8705 bool IllegalFPCMov = false; 8706 if (VT.isFloatingPoint() && !VT.isVector() && 8707 !isScalarFPTypeInSSEReg(VT)) // FPStack? 8708 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue()); 8709 8710 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) || 8711 Opc == X86ISD::BT) { // FIXME 8712 Cond = Cmp; 8713 addTest = false; 8714 } 8715 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO || 8716 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO || 8717 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) && 8718 Cond.getOperand(0).getValueType() != MVT::i8)) { 8719 SDValue LHS = Cond.getOperand(0); 8720 SDValue RHS = Cond.getOperand(1); 8721 unsigned X86Opcode; 8722 unsigned X86Cond; 8723 SDVTList VTs; 8724 switch (CondOpcode) { 8725 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break; 8726 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break; 8727 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break; 8728 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break; 8729 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break; 8730 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break; 8731 default: llvm_unreachable("unexpected overflowing operator"); 8732 } 8733 if (CondOpcode == ISD::UMULO) 8734 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(), 8735 MVT::i32); 8736 else 8737 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); 8738 8739 SDValue X86Op = DAG.getNode(X86Opcode, DL, VTs, LHS, RHS); 8740 8741 if (CondOpcode == ISD::UMULO) 8742 Cond = X86Op.getValue(2); 8743 else 8744 Cond = X86Op.getValue(1); 8745 8746 CC = DAG.getConstant(X86Cond, MVT::i8); 8747 addTest = false; 8748 } 8749 8750 if (addTest) { 8751 // Look pass the truncate. 8752 if (Cond.getOpcode() == ISD::TRUNCATE) 8753 Cond = Cond.getOperand(0); 8754 8755 // We know the result of AND is compared against zero. Try to match 8756 // it to BT. 8757 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { 8758 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG); 8759 if (NewSetCC.getNode()) { 8760 CC = NewSetCC.getOperand(0); 8761 Cond = NewSetCC.getOperand(1); 8762 addTest = false; 8763 } 8764 } 8765 } 8766 8767 if (addTest) { 8768 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 8769 Cond = EmitTest(Cond, X86::COND_NE, DAG); 8770 } 8771 8772 // a < b ? -1 : 0 -> RES = ~setcc_carry 8773 // a < b ? 0 : -1 -> RES = setcc_carry 8774 // a >= b ? -1 : 0 -> RES = setcc_carry 8775 // a >= b ? 0 : -1 -> RES = ~setcc_carry 8776 if (Cond.getOpcode() == X86ISD::CMP) { 8777 Cond = ConvertCmpIfNecessary(Cond, DAG); 8778 unsigned CondCode = cast<ConstantSDNode>(CC)->getZExtValue(); 8779 8780 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) && 8781 (isAllOnes(Op1) || isAllOnes(Op2)) && (isZero(Op1) || isZero(Op2))) { 8782 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), 8783 DAG.getConstant(X86::COND_B, MVT::i8), Cond); 8784 if (isAllOnes(Op1) != (CondCode == X86::COND_B)) 8785 return DAG.getNOT(DL, Res, Res.getValueType()); 8786 return Res; 8787 } 8788 } 8789 8790 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 8791 // condition is true. 8792 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 8793 SDValue Ops[] = { Op2, Op1, CC, Cond }; 8794 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops, array_lengthof(Ops)); 8795} 8796 8797// isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or 8798// ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart 8799// from the AND / OR. 8800static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) { 8801 Opc = Op.getOpcode(); 8802 if (Opc != ISD::OR && Opc != ISD::AND) 8803 return false; 8804 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC && 8805 Op.getOperand(0).hasOneUse() && 8806 Op.getOperand(1).getOpcode() == X86ISD::SETCC && 8807 Op.getOperand(1).hasOneUse()); 8808} 8809 8810// isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and 8811// 1 and that the SETCC node has a single use. 8812static bool isXor1OfSetCC(SDValue Op) { 8813 if (Op.getOpcode() != ISD::XOR) 8814 return false; 8815 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 8816 if (N1C && N1C->getAPIntValue() == 1) { 8817 return Op.getOperand(0).getOpcode() == X86ISD::SETCC && 8818 Op.getOperand(0).hasOneUse(); 8819 } 8820 return false; 8821} 8822 8823SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 8824 bool addTest = true; 8825 SDValue Chain = Op.getOperand(0); 8826 SDValue Cond = Op.getOperand(1); 8827 SDValue Dest = Op.getOperand(2); 8828 DebugLoc dl = Op.getDebugLoc(); 8829 SDValue CC; 8830 bool Inverted = false; 8831 8832 if (Cond.getOpcode() == ISD::SETCC) { 8833 // Check for setcc([su]{add,sub,mul}o == 0). 8834 if (cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ && 8835 isa<ConstantSDNode>(Cond.getOperand(1)) && 8836 cast<ConstantSDNode>(Cond.getOperand(1))->isNullValue() && 8837 Cond.getOperand(0).getResNo() == 1 && 8838 (Cond.getOperand(0).getOpcode() == ISD::SADDO || 8839 Cond.getOperand(0).getOpcode() == ISD::UADDO || 8840 Cond.getOperand(0).getOpcode() == ISD::SSUBO || 8841 Cond.getOperand(0).getOpcode() == ISD::USUBO || 8842 Cond.getOperand(0).getOpcode() == ISD::SMULO || 8843 Cond.getOperand(0).getOpcode() == ISD::UMULO)) { 8844 Inverted = true; 8845 Cond = Cond.getOperand(0); 8846 } else { 8847 SDValue NewCond = LowerSETCC(Cond, DAG); 8848 if (NewCond.getNode()) 8849 Cond = NewCond; 8850 } 8851 } 8852#if 0 8853 // FIXME: LowerXALUO doesn't handle these!! 8854 else if (Cond.getOpcode() == X86ISD::ADD || 8855 Cond.getOpcode() == X86ISD::SUB || 8856 Cond.getOpcode() == X86ISD::SMUL || 8857 Cond.getOpcode() == X86ISD::UMUL) 8858 Cond = LowerXALUO(Cond, DAG); 8859#endif 8860 8861 // Look pass (and (setcc_carry (cmp ...)), 1). 8862 if (Cond.getOpcode() == ISD::AND && 8863 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) { 8864 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1)); 8865 if (C && C->getAPIntValue() == 1) 8866 Cond = Cond.getOperand(0); 8867 } 8868 8869 // If condition flag is set by a X86ISD::CMP, then use it as the condition 8870 // setting operand in place of the X86ISD::SETCC. 8871 unsigned CondOpcode = Cond.getOpcode(); 8872 if (CondOpcode == X86ISD::SETCC || 8873 CondOpcode == X86ISD::SETCC_CARRY) { 8874 CC = Cond.getOperand(0); 8875 8876 SDValue Cmp = Cond.getOperand(1); 8877 unsigned Opc = Cmp.getOpcode(); 8878 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp?? 8879 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) { 8880 Cond = Cmp; 8881 addTest = false; 8882 } else { 8883 switch (cast<ConstantSDNode>(CC)->getZExtValue()) { 8884 default: break; 8885 case X86::COND_O: 8886 case X86::COND_B: 8887 // These can only come from an arithmetic instruction with overflow, 8888 // e.g. SADDO, UADDO. 8889 Cond = Cond.getNode()->getOperand(1); 8890 addTest = false; 8891 break; 8892 } 8893 } 8894 } 8895 CondOpcode = Cond.getOpcode(); 8896 if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO || 8897 CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO || 8898 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) && 8899 Cond.getOperand(0).getValueType() != MVT::i8)) { 8900 SDValue LHS = Cond.getOperand(0); 8901 SDValue RHS = Cond.getOperand(1); 8902 unsigned X86Opcode; 8903 unsigned X86Cond; 8904 SDVTList VTs; 8905 switch (CondOpcode) { 8906 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break; 8907 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break; 8908 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break; 8909 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break; 8910 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break; 8911 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break; 8912 default: llvm_unreachable("unexpected overflowing operator"); 8913 } 8914 if (Inverted) 8915 X86Cond = X86::GetOppositeBranchCondition((X86::CondCode)X86Cond); 8916 if (CondOpcode == ISD::UMULO) 8917 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(), 8918 MVT::i32); 8919 else 8920 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); 8921 8922 SDValue X86Op = DAG.getNode(X86Opcode, dl, VTs, LHS, RHS); 8923 8924 if (CondOpcode == ISD::UMULO) 8925 Cond = X86Op.getValue(2); 8926 else 8927 Cond = X86Op.getValue(1); 8928 8929 CC = DAG.getConstant(X86Cond, MVT::i8); 8930 addTest = false; 8931 } else { 8932 unsigned CondOpc; 8933 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) { 8934 SDValue Cmp = Cond.getOperand(0).getOperand(1); 8935 if (CondOpc == ISD::OR) { 8936 // Also, recognize the pattern generated by an FCMP_UNE. We can emit 8937 // two branches instead of an explicit OR instruction with a 8938 // separate test. 8939 if (Cmp == Cond.getOperand(1).getOperand(1) && 8940 isX86LogicalCmp(Cmp)) { 8941 CC = Cond.getOperand(0).getOperand(0); 8942 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 8943 Chain, Dest, CC, Cmp); 8944 CC = Cond.getOperand(1).getOperand(0); 8945 Cond = Cmp; 8946 addTest = false; 8947 } 8948 } else { // ISD::AND 8949 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit 8950 // two branches instead of an explicit AND instruction with a 8951 // separate test. However, we only do this if this block doesn't 8952 // have a fall-through edge, because this requires an explicit 8953 // jmp when the condition is false. 8954 if (Cmp == Cond.getOperand(1).getOperand(1) && 8955 isX86LogicalCmp(Cmp) && 8956 Op.getNode()->hasOneUse()) { 8957 X86::CondCode CCode = 8958 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 8959 CCode = X86::GetOppositeBranchCondition(CCode); 8960 CC = DAG.getConstant(CCode, MVT::i8); 8961 SDNode *User = *Op.getNode()->use_begin(); 8962 // Look for an unconditional branch following this conditional branch. 8963 // We need this because we need to reverse the successors in order 8964 // to implement FCMP_OEQ. 8965 if (User->getOpcode() == ISD::BR) { 8966 SDValue FalseBB = User->getOperand(1); 8967 SDNode *NewBR = 8968 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); 8969 assert(NewBR == User); 8970 (void)NewBR; 8971 Dest = FalseBB; 8972 8973 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 8974 Chain, Dest, CC, Cmp); 8975 X86::CondCode CCode = 8976 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0); 8977 CCode = X86::GetOppositeBranchCondition(CCode); 8978 CC = DAG.getConstant(CCode, MVT::i8); 8979 Cond = Cmp; 8980 addTest = false; 8981 } 8982 } 8983 } 8984 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) { 8985 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition. 8986 // It should be transformed during dag combiner except when the condition 8987 // is set by a arithmetics with overflow node. 8988 X86::CondCode CCode = 8989 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 8990 CCode = X86::GetOppositeBranchCondition(CCode); 8991 CC = DAG.getConstant(CCode, MVT::i8); 8992 Cond = Cond.getOperand(0).getOperand(1); 8993 addTest = false; 8994 } else if (Cond.getOpcode() == ISD::SETCC && 8995 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETOEQ) { 8996 // For FCMP_OEQ, we can emit 8997 // two branches instead of an explicit AND instruction with a 8998 // separate test. However, we only do this if this block doesn't 8999 // have a fall-through edge, because this requires an explicit 9000 // jmp when the condition is false. 9001 if (Op.getNode()->hasOneUse()) { 9002 SDNode *User = *Op.getNode()->use_begin(); 9003 // Look for an unconditional branch following this conditional branch. 9004 // We need this because we need to reverse the successors in order 9005 // to implement FCMP_OEQ. 9006 if (User->getOpcode() == ISD::BR) { 9007 SDValue FalseBB = User->getOperand(1); 9008 SDNode *NewBR = 9009 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); 9010 assert(NewBR == User); 9011 (void)NewBR; 9012 Dest = FalseBB; 9013 9014 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, 9015 Cond.getOperand(0), Cond.getOperand(1)); 9016 Cmp = ConvertCmpIfNecessary(Cmp, DAG); 9017 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 9018 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 9019 Chain, Dest, CC, Cmp); 9020 CC = DAG.getConstant(X86::COND_P, MVT::i8); 9021 Cond = Cmp; 9022 addTest = false; 9023 } 9024 } 9025 } else if (Cond.getOpcode() == ISD::SETCC && 9026 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) { 9027 // For FCMP_UNE, we can emit 9028 // two branches instead of an explicit AND instruction with a 9029 // separate test. However, we only do this if this block doesn't 9030 // have a fall-through edge, because this requires an explicit 9031 // jmp when the condition is false. 9032 if (Op.getNode()->hasOneUse()) { 9033 SDNode *User = *Op.getNode()->use_begin(); 9034 // Look for an unconditional branch following this conditional branch. 9035 // We need this because we need to reverse the successors in order 9036 // to implement FCMP_UNE. 9037 if (User->getOpcode() == ISD::BR) { 9038 SDValue FalseBB = User->getOperand(1); 9039 SDNode *NewBR = 9040 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); 9041 assert(NewBR == User); 9042 (void)NewBR; 9043 9044 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, 9045 Cond.getOperand(0), Cond.getOperand(1)); 9046 Cmp = ConvertCmpIfNecessary(Cmp, DAG); 9047 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 9048 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 9049 Chain, Dest, CC, Cmp); 9050 CC = DAG.getConstant(X86::COND_NP, MVT::i8); 9051 Cond = Cmp; 9052 addTest = false; 9053 Dest = FalseBB; 9054 } 9055 } 9056 } 9057 } 9058 9059 if (addTest) { 9060 // Look pass the truncate. 9061 if (Cond.getOpcode() == ISD::TRUNCATE) 9062 Cond = Cond.getOperand(0); 9063 9064 // We know the result of AND is compared against zero. Try to match 9065 // it to BT. 9066 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { 9067 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG); 9068 if (NewSetCC.getNode()) { 9069 CC = NewSetCC.getOperand(0); 9070 Cond = NewSetCC.getOperand(1); 9071 addTest = false; 9072 } 9073 } 9074 } 9075 9076 if (addTest) { 9077 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 9078 Cond = EmitTest(Cond, X86::COND_NE, DAG); 9079 } 9080 Cond = ConvertCmpIfNecessary(Cond, DAG); 9081 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 9082 Chain, Dest, CC, Cond); 9083} 9084 9085 9086// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 9087// Calls to _alloca is needed to probe the stack when allocating more than 4k 9088// bytes in one go. Touching the stack at 4K increments is necessary to ensure 9089// that the guard pages used by the OS virtual memory manager are allocated in 9090// correct sequence. 9091SDValue 9092X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 9093 SelectionDAG &DAG) const { 9094 assert((Subtarget->isTargetCygMing() || Subtarget->isTargetWindows() || 9095 getTargetMachine().Options.EnableSegmentedStacks) && 9096 "This should be used only on Windows targets or when segmented stacks " 9097 "are being used"); 9098 assert(!Subtarget->isTargetEnvMacho() && "Not implemented"); 9099 DebugLoc dl = Op.getDebugLoc(); 9100 9101 // Get the inputs. 9102 SDValue Chain = Op.getOperand(0); 9103 SDValue Size = Op.getOperand(1); 9104 // FIXME: Ensure alignment here 9105 9106 bool Is64Bit = Subtarget->is64Bit(); 9107 EVT SPTy = Is64Bit ? MVT::i64 : MVT::i32; 9108 9109 if (getTargetMachine().Options.EnableSegmentedStacks) { 9110 MachineFunction &MF = DAG.getMachineFunction(); 9111 MachineRegisterInfo &MRI = MF.getRegInfo(); 9112 9113 if (Is64Bit) { 9114 // The 64 bit implementation of segmented stacks needs to clobber both r10 9115 // r11. This makes it impossible to use it along with nested parameters. 9116 const Function *F = MF.getFunction(); 9117 9118 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end(); 9119 I != E; ++I) 9120 if (I->hasNestAttr()) 9121 report_fatal_error("Cannot use segmented stacks with functions that " 9122 "have nested arguments."); 9123 } 9124 9125 const TargetRegisterClass *AddrRegClass = 9126 getRegClassFor(Subtarget->is64Bit() ? MVT::i64:MVT::i32); 9127 unsigned Vreg = MRI.createVirtualRegister(AddrRegClass); 9128 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size); 9129 SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain, 9130 DAG.getRegister(Vreg, SPTy)); 9131 SDValue Ops1[2] = { Value, Chain }; 9132 return DAG.getMergeValues(Ops1, 2, dl); 9133 } else { 9134 SDValue Flag; 9135 unsigned Reg = (Subtarget->is64Bit() ? X86::RAX : X86::EAX); 9136 9137 Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag); 9138 Flag = Chain.getValue(1); 9139 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 9140 9141 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag); 9142 Flag = Chain.getValue(1); 9143 9144 Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1); 9145 9146 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 9147 return DAG.getMergeValues(Ops1, 2, dl); 9148 } 9149} 9150 9151SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { 9152 MachineFunction &MF = DAG.getMachineFunction(); 9153 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 9154 9155 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 9156 DebugLoc DL = Op.getDebugLoc(); 9157 9158 if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) { 9159 // vastart just stores the address of the VarArgsFrameIndex slot into the 9160 // memory location argument. 9161 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 9162 getPointerTy()); 9163 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), 9164 MachinePointerInfo(SV), false, false, 0); 9165 } 9166 9167 // __va_list_tag: 9168 // gp_offset (0 - 6 * 8) 9169 // fp_offset (48 - 48 + 8 * 16) 9170 // overflow_arg_area (point to parameters coming in memory). 9171 // reg_save_area 9172 SmallVector<SDValue, 8> MemOps; 9173 SDValue FIN = Op.getOperand(1); 9174 // Store gp_offset 9175 SDValue Store = DAG.getStore(Op.getOperand(0), DL, 9176 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), 9177 MVT::i32), 9178 FIN, MachinePointerInfo(SV), false, false, 0); 9179 MemOps.push_back(Store); 9180 9181 // Store fp_offset 9182 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 9183 FIN, DAG.getIntPtrConstant(4)); 9184 Store = DAG.getStore(Op.getOperand(0), DL, 9185 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), 9186 MVT::i32), 9187 FIN, MachinePointerInfo(SV, 4), false, false, 0); 9188 MemOps.push_back(Store); 9189 9190 // Store ptr to overflow_arg_area 9191 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 9192 FIN, DAG.getIntPtrConstant(4)); 9193 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 9194 getPointerTy()); 9195 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, 9196 MachinePointerInfo(SV, 8), 9197 false, false, 0); 9198 MemOps.push_back(Store); 9199 9200 // Store ptr to reg_save_area. 9201 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), 9202 FIN, DAG.getIntPtrConstant(8)); 9203 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), 9204 getPointerTy()); 9205 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN, 9206 MachinePointerInfo(SV, 16), false, false, 0); 9207 MemOps.push_back(Store); 9208 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 9209 &MemOps[0], MemOps.size()); 9210} 9211 9212SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 9213 assert(Subtarget->is64Bit() && 9214 "LowerVAARG only handles 64-bit va_arg!"); 9215 assert((Subtarget->isTargetLinux() || 9216 Subtarget->isTargetDarwin()) && 9217 "Unhandled target in LowerVAARG"); 9218 assert(Op.getNode()->getNumOperands() == 4); 9219 SDValue Chain = Op.getOperand(0); 9220 SDValue SrcPtr = Op.getOperand(1); 9221 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 9222 unsigned Align = Op.getConstantOperandVal(3); 9223 DebugLoc dl = Op.getDebugLoc(); 9224 9225 EVT ArgVT = Op.getNode()->getValueType(0); 9226 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 9227 uint32_t ArgSize = getTargetData()->getTypeAllocSize(ArgTy); 9228 uint8_t ArgMode; 9229 9230 // Decide which area this value should be read from. 9231 // TODO: Implement the AMD64 ABI in its entirety. This simple 9232 // selection mechanism works only for the basic types. 9233 if (ArgVT == MVT::f80) { 9234 llvm_unreachable("va_arg for f80 not yet implemented"); 9235 } else if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) { 9236 ArgMode = 2; // Argument passed in XMM register. Use fp_offset. 9237 } else if (ArgVT.isInteger() && ArgSize <= 32 /*bytes*/) { 9238 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset. 9239 } else { 9240 llvm_unreachable("Unhandled argument type in LowerVAARG"); 9241 } 9242 9243 if (ArgMode == 2) { 9244 // Sanity Check: Make sure using fp_offset makes sense. 9245 assert(!getTargetMachine().Options.UseSoftFloat && 9246 !(DAG.getMachineFunction() 9247 .getFunction()->hasFnAttr(Attribute::NoImplicitFloat)) && 9248 Subtarget->hasSSE1()); 9249 } 9250 9251 // Insert VAARG_64 node into the DAG 9252 // VAARG_64 returns two values: Variable Argument Address, Chain 9253 SmallVector<SDValue, 11> InstOps; 9254 InstOps.push_back(Chain); 9255 InstOps.push_back(SrcPtr); 9256 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32)); 9257 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8)); 9258 InstOps.push_back(DAG.getConstant(Align, MVT::i32)); 9259 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other); 9260 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl, 9261 VTs, &InstOps[0], InstOps.size(), 9262 MVT::i64, 9263 MachinePointerInfo(SV), 9264 /*Align=*/0, 9265 /*Volatile=*/false, 9266 /*ReadMem=*/true, 9267 /*WriteMem=*/true); 9268 Chain = VAARG.getValue(1); 9269 9270 // Load the next argument and return it 9271 return DAG.getLoad(ArgVT, dl, 9272 Chain, 9273 VAARG, 9274 MachinePointerInfo(), 9275 false, false, false, 0); 9276} 9277 9278SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { 9279 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 9280 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 9281 SDValue Chain = Op.getOperand(0); 9282 SDValue DstPtr = Op.getOperand(1); 9283 SDValue SrcPtr = Op.getOperand(2); 9284 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 9285 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 9286 DebugLoc DL = Op.getDebugLoc(); 9287 9288 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, 9289 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false, 9290 false, 9291 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 9292} 9293 9294// getTargetVShiftNOde - Handle vector element shifts where the shift amount 9295// may or may not be a constant. Takes immediate version of shift as input. 9296static SDValue getTargetVShiftNode(unsigned Opc, DebugLoc dl, EVT VT, 9297 SDValue SrcOp, SDValue ShAmt, 9298 SelectionDAG &DAG) { 9299 assert(ShAmt.getValueType() == MVT::i32 && "ShAmt is not i32"); 9300 9301 if (isa<ConstantSDNode>(ShAmt)) { 9302 switch (Opc) { 9303 default: llvm_unreachable("Unknown target vector shift node"); 9304 case X86ISD::VSHLI: 9305 case X86ISD::VSRLI: 9306 case X86ISD::VSRAI: 9307 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); 9308 } 9309 } 9310 9311 // Change opcode to non-immediate version 9312 switch (Opc) { 9313 default: llvm_unreachable("Unknown target vector shift node"); 9314 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break; 9315 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break; 9316 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break; 9317 } 9318 9319 // Need to build a vector containing shift amount 9320 // Shift amount is 32-bits, but SSE instructions read 64-bit, so fill with 0 9321 SDValue ShOps[4]; 9322 ShOps[0] = ShAmt; 9323 ShOps[1] = DAG.getConstant(0, MVT::i32); 9324 ShOps[2] = DAG.getUNDEF(MVT::i32); 9325 ShOps[3] = DAG.getUNDEF(MVT::i32); 9326 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, &ShOps[0], 4); 9327 ShAmt = DAG.getNode(ISD::BITCAST, dl, VT, ShAmt); 9328 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); 9329} 9330 9331SDValue 9332X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { 9333 DebugLoc dl = Op.getDebugLoc(); 9334 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9335 switch (IntNo) { 9336 default: return SDValue(); // Don't custom lower most intrinsics. 9337 // Comparison intrinsics. 9338 case Intrinsic::x86_sse_comieq_ss: 9339 case Intrinsic::x86_sse_comilt_ss: 9340 case Intrinsic::x86_sse_comile_ss: 9341 case Intrinsic::x86_sse_comigt_ss: 9342 case Intrinsic::x86_sse_comige_ss: 9343 case Intrinsic::x86_sse_comineq_ss: 9344 case Intrinsic::x86_sse_ucomieq_ss: 9345 case Intrinsic::x86_sse_ucomilt_ss: 9346 case Intrinsic::x86_sse_ucomile_ss: 9347 case Intrinsic::x86_sse_ucomigt_ss: 9348 case Intrinsic::x86_sse_ucomige_ss: 9349 case Intrinsic::x86_sse_ucomineq_ss: 9350 case Intrinsic::x86_sse2_comieq_sd: 9351 case Intrinsic::x86_sse2_comilt_sd: 9352 case Intrinsic::x86_sse2_comile_sd: 9353 case Intrinsic::x86_sse2_comigt_sd: 9354 case Intrinsic::x86_sse2_comige_sd: 9355 case Intrinsic::x86_sse2_comineq_sd: 9356 case Intrinsic::x86_sse2_ucomieq_sd: 9357 case Intrinsic::x86_sse2_ucomilt_sd: 9358 case Intrinsic::x86_sse2_ucomile_sd: 9359 case Intrinsic::x86_sse2_ucomigt_sd: 9360 case Intrinsic::x86_sse2_ucomige_sd: 9361 case Intrinsic::x86_sse2_ucomineq_sd: { 9362 unsigned Opc = 0; 9363 ISD::CondCode CC = ISD::SETCC_INVALID; 9364 switch (IntNo) { 9365 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 9366 case Intrinsic::x86_sse_comieq_ss: 9367 case Intrinsic::x86_sse2_comieq_sd: 9368 Opc = X86ISD::COMI; 9369 CC = ISD::SETEQ; 9370 break; 9371 case Intrinsic::x86_sse_comilt_ss: 9372 case Intrinsic::x86_sse2_comilt_sd: 9373 Opc = X86ISD::COMI; 9374 CC = ISD::SETLT; 9375 break; 9376 case Intrinsic::x86_sse_comile_ss: 9377 case Intrinsic::x86_sse2_comile_sd: 9378 Opc = X86ISD::COMI; 9379 CC = ISD::SETLE; 9380 break; 9381 case Intrinsic::x86_sse_comigt_ss: 9382 case Intrinsic::x86_sse2_comigt_sd: 9383 Opc = X86ISD::COMI; 9384 CC = ISD::SETGT; 9385 break; 9386 case Intrinsic::x86_sse_comige_ss: 9387 case Intrinsic::x86_sse2_comige_sd: 9388 Opc = X86ISD::COMI; 9389 CC = ISD::SETGE; 9390 break; 9391 case Intrinsic::x86_sse_comineq_ss: 9392 case Intrinsic::x86_sse2_comineq_sd: 9393 Opc = X86ISD::COMI; 9394 CC = ISD::SETNE; 9395 break; 9396 case Intrinsic::x86_sse_ucomieq_ss: 9397 case Intrinsic::x86_sse2_ucomieq_sd: 9398 Opc = X86ISD::UCOMI; 9399 CC = ISD::SETEQ; 9400 break; 9401 case Intrinsic::x86_sse_ucomilt_ss: 9402 case Intrinsic::x86_sse2_ucomilt_sd: 9403 Opc = X86ISD::UCOMI; 9404 CC = ISD::SETLT; 9405 break; 9406 case Intrinsic::x86_sse_ucomile_ss: 9407 case Intrinsic::x86_sse2_ucomile_sd: 9408 Opc = X86ISD::UCOMI; 9409 CC = ISD::SETLE; 9410 break; 9411 case Intrinsic::x86_sse_ucomigt_ss: 9412 case Intrinsic::x86_sse2_ucomigt_sd: 9413 Opc = X86ISD::UCOMI; 9414 CC = ISD::SETGT; 9415 break; 9416 case Intrinsic::x86_sse_ucomige_ss: 9417 case Intrinsic::x86_sse2_ucomige_sd: 9418 Opc = X86ISD::UCOMI; 9419 CC = ISD::SETGE; 9420 break; 9421 case Intrinsic::x86_sse_ucomineq_ss: 9422 case Intrinsic::x86_sse2_ucomineq_sd: 9423 Opc = X86ISD::UCOMI; 9424 CC = ISD::SETNE; 9425 break; 9426 } 9427 9428 SDValue LHS = Op.getOperand(1); 9429 SDValue RHS = Op.getOperand(2); 9430 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG); 9431 assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!"); 9432 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS); 9433 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 9434 DAG.getConstant(X86CC, MVT::i8), Cond); 9435 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 9436 } 9437 // XOP comparison intrinsics 9438 case Intrinsic::x86_xop_vpcomltb: 9439 case Intrinsic::x86_xop_vpcomltw: 9440 case Intrinsic::x86_xop_vpcomltd: 9441 case Intrinsic::x86_xop_vpcomltq: 9442 case Intrinsic::x86_xop_vpcomltub: 9443 case Intrinsic::x86_xop_vpcomltuw: 9444 case Intrinsic::x86_xop_vpcomltud: 9445 case Intrinsic::x86_xop_vpcomltuq: 9446 case Intrinsic::x86_xop_vpcomleb: 9447 case Intrinsic::x86_xop_vpcomlew: 9448 case Intrinsic::x86_xop_vpcomled: 9449 case Intrinsic::x86_xop_vpcomleq: 9450 case Intrinsic::x86_xop_vpcomleub: 9451 case Intrinsic::x86_xop_vpcomleuw: 9452 case Intrinsic::x86_xop_vpcomleud: 9453 case Intrinsic::x86_xop_vpcomleuq: 9454 case Intrinsic::x86_xop_vpcomgtb: 9455 case Intrinsic::x86_xop_vpcomgtw: 9456 case Intrinsic::x86_xop_vpcomgtd: 9457 case Intrinsic::x86_xop_vpcomgtq: 9458 case Intrinsic::x86_xop_vpcomgtub: 9459 case Intrinsic::x86_xop_vpcomgtuw: 9460 case Intrinsic::x86_xop_vpcomgtud: 9461 case Intrinsic::x86_xop_vpcomgtuq: 9462 case Intrinsic::x86_xop_vpcomgeb: 9463 case Intrinsic::x86_xop_vpcomgew: 9464 case Intrinsic::x86_xop_vpcomged: 9465 case Intrinsic::x86_xop_vpcomgeq: 9466 case Intrinsic::x86_xop_vpcomgeub: 9467 case Intrinsic::x86_xop_vpcomgeuw: 9468 case Intrinsic::x86_xop_vpcomgeud: 9469 case Intrinsic::x86_xop_vpcomgeuq: 9470 case Intrinsic::x86_xop_vpcomeqb: 9471 case Intrinsic::x86_xop_vpcomeqw: 9472 case Intrinsic::x86_xop_vpcomeqd: 9473 case Intrinsic::x86_xop_vpcomeqq: 9474 case Intrinsic::x86_xop_vpcomequb: 9475 case Intrinsic::x86_xop_vpcomequw: 9476 case Intrinsic::x86_xop_vpcomequd: 9477 case Intrinsic::x86_xop_vpcomequq: 9478 case Intrinsic::x86_xop_vpcomneb: 9479 case Intrinsic::x86_xop_vpcomnew: 9480 case Intrinsic::x86_xop_vpcomned: 9481 case Intrinsic::x86_xop_vpcomneq: 9482 case Intrinsic::x86_xop_vpcomneub: 9483 case Intrinsic::x86_xop_vpcomneuw: 9484 case Intrinsic::x86_xop_vpcomneud: 9485 case Intrinsic::x86_xop_vpcomneuq: 9486 case Intrinsic::x86_xop_vpcomfalseb: 9487 case Intrinsic::x86_xop_vpcomfalsew: 9488 case Intrinsic::x86_xop_vpcomfalsed: 9489 case Intrinsic::x86_xop_vpcomfalseq: 9490 case Intrinsic::x86_xop_vpcomfalseub: 9491 case Intrinsic::x86_xop_vpcomfalseuw: 9492 case Intrinsic::x86_xop_vpcomfalseud: 9493 case Intrinsic::x86_xop_vpcomfalseuq: 9494 case Intrinsic::x86_xop_vpcomtrueb: 9495 case Intrinsic::x86_xop_vpcomtruew: 9496 case Intrinsic::x86_xop_vpcomtrued: 9497 case Intrinsic::x86_xop_vpcomtrueq: 9498 case Intrinsic::x86_xop_vpcomtrueub: 9499 case Intrinsic::x86_xop_vpcomtrueuw: 9500 case Intrinsic::x86_xop_vpcomtrueud: 9501 case Intrinsic::x86_xop_vpcomtrueuq: { 9502 unsigned CC = 0; 9503 unsigned Opc = 0; 9504 9505 switch (IntNo) { 9506 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 9507 case Intrinsic::x86_xop_vpcomltb: 9508 case Intrinsic::x86_xop_vpcomltw: 9509 case Intrinsic::x86_xop_vpcomltd: 9510 case Intrinsic::x86_xop_vpcomltq: 9511 CC = 0; 9512 Opc = X86ISD::VPCOM; 9513 break; 9514 case Intrinsic::x86_xop_vpcomltub: 9515 case Intrinsic::x86_xop_vpcomltuw: 9516 case Intrinsic::x86_xop_vpcomltud: 9517 case Intrinsic::x86_xop_vpcomltuq: 9518 CC = 0; 9519 Opc = X86ISD::VPCOMU; 9520 break; 9521 case Intrinsic::x86_xop_vpcomleb: 9522 case Intrinsic::x86_xop_vpcomlew: 9523 case Intrinsic::x86_xop_vpcomled: 9524 case Intrinsic::x86_xop_vpcomleq: 9525 CC = 1; 9526 Opc = X86ISD::VPCOM; 9527 break; 9528 case Intrinsic::x86_xop_vpcomleub: 9529 case Intrinsic::x86_xop_vpcomleuw: 9530 case Intrinsic::x86_xop_vpcomleud: 9531 case Intrinsic::x86_xop_vpcomleuq: 9532 CC = 1; 9533 Opc = X86ISD::VPCOMU; 9534 break; 9535 case Intrinsic::x86_xop_vpcomgtb: 9536 case Intrinsic::x86_xop_vpcomgtw: 9537 case Intrinsic::x86_xop_vpcomgtd: 9538 case Intrinsic::x86_xop_vpcomgtq: 9539 CC = 2; 9540 Opc = X86ISD::VPCOM; 9541 break; 9542 case Intrinsic::x86_xop_vpcomgtub: 9543 case Intrinsic::x86_xop_vpcomgtuw: 9544 case Intrinsic::x86_xop_vpcomgtud: 9545 case Intrinsic::x86_xop_vpcomgtuq: 9546 CC = 2; 9547 Opc = X86ISD::VPCOMU; 9548 break; 9549 case Intrinsic::x86_xop_vpcomgeb: 9550 case Intrinsic::x86_xop_vpcomgew: 9551 case Intrinsic::x86_xop_vpcomged: 9552 case Intrinsic::x86_xop_vpcomgeq: 9553 CC = 3; 9554 Opc = X86ISD::VPCOM; 9555 break; 9556 case Intrinsic::x86_xop_vpcomgeub: 9557 case Intrinsic::x86_xop_vpcomgeuw: 9558 case Intrinsic::x86_xop_vpcomgeud: 9559 case Intrinsic::x86_xop_vpcomgeuq: 9560 CC = 3; 9561 Opc = X86ISD::VPCOMU; 9562 break; 9563 case Intrinsic::x86_xop_vpcomeqb: 9564 case Intrinsic::x86_xop_vpcomeqw: 9565 case Intrinsic::x86_xop_vpcomeqd: 9566 case Intrinsic::x86_xop_vpcomeqq: 9567 CC = 4; 9568 Opc = X86ISD::VPCOM; 9569 break; 9570 case Intrinsic::x86_xop_vpcomequb: 9571 case Intrinsic::x86_xop_vpcomequw: 9572 case Intrinsic::x86_xop_vpcomequd: 9573 case Intrinsic::x86_xop_vpcomequq: 9574 CC = 4; 9575 Opc = X86ISD::VPCOMU; 9576 break; 9577 case Intrinsic::x86_xop_vpcomneb: 9578 case Intrinsic::x86_xop_vpcomnew: 9579 case Intrinsic::x86_xop_vpcomned: 9580 case Intrinsic::x86_xop_vpcomneq: 9581 CC = 5; 9582 Opc = X86ISD::VPCOM; 9583 break; 9584 case Intrinsic::x86_xop_vpcomneub: 9585 case Intrinsic::x86_xop_vpcomneuw: 9586 case Intrinsic::x86_xop_vpcomneud: 9587 case Intrinsic::x86_xop_vpcomneuq: 9588 CC = 5; 9589 Opc = X86ISD::VPCOMU; 9590 break; 9591 case Intrinsic::x86_xop_vpcomfalseb: 9592 case Intrinsic::x86_xop_vpcomfalsew: 9593 case Intrinsic::x86_xop_vpcomfalsed: 9594 case Intrinsic::x86_xop_vpcomfalseq: 9595 CC = 6; 9596 Opc = X86ISD::VPCOM; 9597 break; 9598 case Intrinsic::x86_xop_vpcomfalseub: 9599 case Intrinsic::x86_xop_vpcomfalseuw: 9600 case Intrinsic::x86_xop_vpcomfalseud: 9601 case Intrinsic::x86_xop_vpcomfalseuq: 9602 CC = 6; 9603 Opc = X86ISD::VPCOMU; 9604 break; 9605 case Intrinsic::x86_xop_vpcomtrueb: 9606 case Intrinsic::x86_xop_vpcomtruew: 9607 case Intrinsic::x86_xop_vpcomtrued: 9608 case Intrinsic::x86_xop_vpcomtrueq: 9609 CC = 7; 9610 Opc = X86ISD::VPCOM; 9611 break; 9612 case Intrinsic::x86_xop_vpcomtrueub: 9613 case Intrinsic::x86_xop_vpcomtrueuw: 9614 case Intrinsic::x86_xop_vpcomtrueud: 9615 case Intrinsic::x86_xop_vpcomtrueuq: 9616 CC = 7; 9617 Opc = X86ISD::VPCOMU; 9618 break; 9619 } 9620 9621 SDValue LHS = Op.getOperand(1); 9622 SDValue RHS = Op.getOperand(2); 9623 return DAG.getNode(Opc, dl, Op.getValueType(), LHS, RHS, 9624 DAG.getConstant(CC, MVT::i8)); 9625 } 9626 9627 // Arithmetic intrinsics. 9628 case Intrinsic::x86_sse2_pmulu_dq: 9629 case Intrinsic::x86_avx2_pmulu_dq: 9630 return DAG.getNode(X86ISD::PMULUDQ, dl, Op.getValueType(), 9631 Op.getOperand(1), Op.getOperand(2)); 9632 case Intrinsic::x86_sse3_hadd_ps: 9633 case Intrinsic::x86_sse3_hadd_pd: 9634 case Intrinsic::x86_avx_hadd_ps_256: 9635 case Intrinsic::x86_avx_hadd_pd_256: 9636 return DAG.getNode(X86ISD::FHADD, dl, Op.getValueType(), 9637 Op.getOperand(1), Op.getOperand(2)); 9638 case Intrinsic::x86_sse3_hsub_ps: 9639 case Intrinsic::x86_sse3_hsub_pd: 9640 case Intrinsic::x86_avx_hsub_ps_256: 9641 case Intrinsic::x86_avx_hsub_pd_256: 9642 return DAG.getNode(X86ISD::FHSUB, dl, Op.getValueType(), 9643 Op.getOperand(1), Op.getOperand(2)); 9644 case Intrinsic::x86_ssse3_phadd_w_128: 9645 case Intrinsic::x86_ssse3_phadd_d_128: 9646 case Intrinsic::x86_avx2_phadd_w: 9647 case Intrinsic::x86_avx2_phadd_d: 9648 return DAG.getNode(X86ISD::HADD, dl, Op.getValueType(), 9649 Op.getOperand(1), Op.getOperand(2)); 9650 case Intrinsic::x86_ssse3_phsub_w_128: 9651 case Intrinsic::x86_ssse3_phsub_d_128: 9652 case Intrinsic::x86_avx2_phsub_w: 9653 case Intrinsic::x86_avx2_phsub_d: 9654 return DAG.getNode(X86ISD::HSUB, dl, Op.getValueType(), 9655 Op.getOperand(1), Op.getOperand(2)); 9656 case Intrinsic::x86_avx2_psllv_d: 9657 case Intrinsic::x86_avx2_psllv_q: 9658 case Intrinsic::x86_avx2_psllv_d_256: 9659 case Intrinsic::x86_avx2_psllv_q_256: 9660 return DAG.getNode(ISD::SHL, dl, Op.getValueType(), 9661 Op.getOperand(1), Op.getOperand(2)); 9662 case Intrinsic::x86_avx2_psrlv_d: 9663 case Intrinsic::x86_avx2_psrlv_q: 9664 case Intrinsic::x86_avx2_psrlv_d_256: 9665 case Intrinsic::x86_avx2_psrlv_q_256: 9666 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), 9667 Op.getOperand(1), Op.getOperand(2)); 9668 case Intrinsic::x86_avx2_psrav_d: 9669 case Intrinsic::x86_avx2_psrav_d_256: 9670 return DAG.getNode(ISD::SRA, dl, Op.getValueType(), 9671 Op.getOperand(1), Op.getOperand(2)); 9672 case Intrinsic::x86_ssse3_pshuf_b_128: 9673 case Intrinsic::x86_avx2_pshuf_b: 9674 return DAG.getNode(X86ISD::PSHUFB, dl, Op.getValueType(), 9675 Op.getOperand(1), Op.getOperand(2)); 9676 case Intrinsic::x86_ssse3_psign_b_128: 9677 case Intrinsic::x86_ssse3_psign_w_128: 9678 case Intrinsic::x86_ssse3_psign_d_128: 9679 case Intrinsic::x86_avx2_psign_b: 9680 case Intrinsic::x86_avx2_psign_w: 9681 case Intrinsic::x86_avx2_psign_d: 9682 return DAG.getNode(X86ISD::PSIGN, dl, Op.getValueType(), 9683 Op.getOperand(1), Op.getOperand(2)); 9684 case Intrinsic::x86_sse41_insertps: 9685 return DAG.getNode(X86ISD::INSERTPS, dl, Op.getValueType(), 9686 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 9687 case Intrinsic::x86_avx_vperm2f128_ps_256: 9688 case Intrinsic::x86_avx_vperm2f128_pd_256: 9689 case Intrinsic::x86_avx_vperm2f128_si_256: 9690 case Intrinsic::x86_avx2_vperm2i128: 9691 return DAG.getNode(X86ISD::VPERM2X128, dl, Op.getValueType(), 9692 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 9693 case Intrinsic::x86_avx2_permd: 9694 case Intrinsic::x86_avx2_permps: 9695 // Operands intentionally swapped. Mask is last operand to intrinsic, 9696 // but second operand for node/intruction. 9697 return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(), 9698 Op.getOperand(2), Op.getOperand(1)); 9699 9700 // ptest and testp intrinsics. The intrinsic these come from are designed to 9701 // return an integer value, not just an instruction so lower it to the ptest 9702 // or testp pattern and a setcc for the result. 9703 case Intrinsic::x86_sse41_ptestz: 9704 case Intrinsic::x86_sse41_ptestc: 9705 case Intrinsic::x86_sse41_ptestnzc: 9706 case Intrinsic::x86_avx_ptestz_256: 9707 case Intrinsic::x86_avx_ptestc_256: 9708 case Intrinsic::x86_avx_ptestnzc_256: 9709 case Intrinsic::x86_avx_vtestz_ps: 9710 case Intrinsic::x86_avx_vtestc_ps: 9711 case Intrinsic::x86_avx_vtestnzc_ps: 9712 case Intrinsic::x86_avx_vtestz_pd: 9713 case Intrinsic::x86_avx_vtestc_pd: 9714 case Intrinsic::x86_avx_vtestnzc_pd: 9715 case Intrinsic::x86_avx_vtestz_ps_256: 9716 case Intrinsic::x86_avx_vtestc_ps_256: 9717 case Intrinsic::x86_avx_vtestnzc_ps_256: 9718 case Intrinsic::x86_avx_vtestz_pd_256: 9719 case Intrinsic::x86_avx_vtestc_pd_256: 9720 case Intrinsic::x86_avx_vtestnzc_pd_256: { 9721 bool IsTestPacked = false; 9722 unsigned X86CC = 0; 9723 switch (IntNo) { 9724 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering."); 9725 case Intrinsic::x86_avx_vtestz_ps: 9726 case Intrinsic::x86_avx_vtestz_pd: 9727 case Intrinsic::x86_avx_vtestz_ps_256: 9728 case Intrinsic::x86_avx_vtestz_pd_256: 9729 IsTestPacked = true; // Fallthrough 9730 case Intrinsic::x86_sse41_ptestz: 9731 case Intrinsic::x86_avx_ptestz_256: 9732 // ZF = 1 9733 X86CC = X86::COND_E; 9734 break; 9735 case Intrinsic::x86_avx_vtestc_ps: 9736 case Intrinsic::x86_avx_vtestc_pd: 9737 case Intrinsic::x86_avx_vtestc_ps_256: 9738 case Intrinsic::x86_avx_vtestc_pd_256: 9739 IsTestPacked = true; // Fallthrough 9740 case Intrinsic::x86_sse41_ptestc: 9741 case Intrinsic::x86_avx_ptestc_256: 9742 // CF = 1 9743 X86CC = X86::COND_B; 9744 break; 9745 case Intrinsic::x86_avx_vtestnzc_ps: 9746 case Intrinsic::x86_avx_vtestnzc_pd: 9747 case Intrinsic::x86_avx_vtestnzc_ps_256: 9748 case Intrinsic::x86_avx_vtestnzc_pd_256: 9749 IsTestPacked = true; // Fallthrough 9750 case Intrinsic::x86_sse41_ptestnzc: 9751 case Intrinsic::x86_avx_ptestnzc_256: 9752 // ZF and CF = 0 9753 X86CC = X86::COND_A; 9754 break; 9755 } 9756 9757 SDValue LHS = Op.getOperand(1); 9758 SDValue RHS = Op.getOperand(2); 9759 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST; 9760 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); 9761 SDValue CC = DAG.getConstant(X86CC, MVT::i8); 9762 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); 9763 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 9764 } 9765 9766 // SSE/AVX shift intrinsics 9767 case Intrinsic::x86_sse2_psll_w: 9768 case Intrinsic::x86_sse2_psll_d: 9769 case Intrinsic::x86_sse2_psll_q: 9770 case Intrinsic::x86_avx2_psll_w: 9771 case Intrinsic::x86_avx2_psll_d: 9772 case Intrinsic::x86_avx2_psll_q: 9773 return DAG.getNode(X86ISD::VSHL, dl, Op.getValueType(), 9774 Op.getOperand(1), Op.getOperand(2)); 9775 case Intrinsic::x86_sse2_psrl_w: 9776 case Intrinsic::x86_sse2_psrl_d: 9777 case Intrinsic::x86_sse2_psrl_q: 9778 case Intrinsic::x86_avx2_psrl_w: 9779 case Intrinsic::x86_avx2_psrl_d: 9780 case Intrinsic::x86_avx2_psrl_q: 9781 return DAG.getNode(X86ISD::VSRL, dl, Op.getValueType(), 9782 Op.getOperand(1), Op.getOperand(2)); 9783 case Intrinsic::x86_sse2_psra_w: 9784 case Intrinsic::x86_sse2_psra_d: 9785 case Intrinsic::x86_avx2_psra_w: 9786 case Intrinsic::x86_avx2_psra_d: 9787 return DAG.getNode(X86ISD::VSRA, dl, Op.getValueType(), 9788 Op.getOperand(1), Op.getOperand(2)); 9789 case Intrinsic::x86_sse2_pslli_w: 9790 case Intrinsic::x86_sse2_pslli_d: 9791 case Intrinsic::x86_sse2_pslli_q: 9792 case Intrinsic::x86_avx2_pslli_w: 9793 case Intrinsic::x86_avx2_pslli_d: 9794 case Intrinsic::x86_avx2_pslli_q: 9795 return getTargetVShiftNode(X86ISD::VSHLI, dl, Op.getValueType(), 9796 Op.getOperand(1), Op.getOperand(2), DAG); 9797 case Intrinsic::x86_sse2_psrli_w: 9798 case Intrinsic::x86_sse2_psrli_d: 9799 case Intrinsic::x86_sse2_psrli_q: 9800 case Intrinsic::x86_avx2_psrli_w: 9801 case Intrinsic::x86_avx2_psrli_d: 9802 case Intrinsic::x86_avx2_psrli_q: 9803 return getTargetVShiftNode(X86ISD::VSRLI, dl, Op.getValueType(), 9804 Op.getOperand(1), Op.getOperand(2), DAG); 9805 case Intrinsic::x86_sse2_psrai_w: 9806 case Intrinsic::x86_sse2_psrai_d: 9807 case Intrinsic::x86_avx2_psrai_w: 9808 case Intrinsic::x86_avx2_psrai_d: 9809 return getTargetVShiftNode(X86ISD::VSRAI, dl, Op.getValueType(), 9810 Op.getOperand(1), Op.getOperand(2), DAG); 9811 // Fix vector shift instructions where the last operand is a non-immediate 9812 // i32 value. 9813 case Intrinsic::x86_mmx_pslli_w: 9814 case Intrinsic::x86_mmx_pslli_d: 9815 case Intrinsic::x86_mmx_pslli_q: 9816 case Intrinsic::x86_mmx_psrli_w: 9817 case Intrinsic::x86_mmx_psrli_d: 9818 case Intrinsic::x86_mmx_psrli_q: 9819 case Intrinsic::x86_mmx_psrai_w: 9820 case Intrinsic::x86_mmx_psrai_d: { 9821 SDValue ShAmt = Op.getOperand(2); 9822 if (isa<ConstantSDNode>(ShAmt)) 9823 return SDValue(); 9824 9825 unsigned NewIntNo = 0; 9826 switch (IntNo) { 9827 case Intrinsic::x86_mmx_pslli_w: 9828 NewIntNo = Intrinsic::x86_mmx_psll_w; 9829 break; 9830 case Intrinsic::x86_mmx_pslli_d: 9831 NewIntNo = Intrinsic::x86_mmx_psll_d; 9832 break; 9833 case Intrinsic::x86_mmx_pslli_q: 9834 NewIntNo = Intrinsic::x86_mmx_psll_q; 9835 break; 9836 case Intrinsic::x86_mmx_psrli_w: 9837 NewIntNo = Intrinsic::x86_mmx_psrl_w; 9838 break; 9839 case Intrinsic::x86_mmx_psrli_d: 9840 NewIntNo = Intrinsic::x86_mmx_psrl_d; 9841 break; 9842 case Intrinsic::x86_mmx_psrli_q: 9843 NewIntNo = Intrinsic::x86_mmx_psrl_q; 9844 break; 9845 case Intrinsic::x86_mmx_psrai_w: 9846 NewIntNo = Intrinsic::x86_mmx_psra_w; 9847 break; 9848 case Intrinsic::x86_mmx_psrai_d: 9849 NewIntNo = Intrinsic::x86_mmx_psra_d; 9850 break; 9851 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 9852 } 9853 9854 // The vector shift intrinsics with scalars uses 32b shift amounts but 9855 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits 9856 // to be zero. 9857 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, ShAmt, 9858 DAG.getConstant(0, MVT::i32)); 9859// FIXME this must be lowered to get rid of the invalid type. 9860 9861 EVT VT = Op.getValueType(); 9862 ShAmt = DAG.getNode(ISD::BITCAST, dl, VT, ShAmt); 9863 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 9864 DAG.getConstant(NewIntNo, MVT::i32), 9865 Op.getOperand(1), ShAmt); 9866 } 9867 } 9868} 9869 9870SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, 9871 SelectionDAG &DAG) const { 9872 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 9873 MFI->setReturnAddressIsTaken(true); 9874 9875 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9876 DebugLoc dl = Op.getDebugLoc(); 9877 9878 if (Depth > 0) { 9879 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 9880 SDValue Offset = 9881 DAG.getConstant(TD->getPointerSize(), 9882 Subtarget->is64Bit() ? MVT::i64 : MVT::i32); 9883 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 9884 DAG.getNode(ISD::ADD, dl, getPointerTy(), 9885 FrameAddr, Offset), 9886 MachinePointerInfo(), false, false, false, 0); 9887 } 9888 9889 // Just load the return address. 9890 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 9891 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 9892 RetAddrFI, MachinePointerInfo(), false, false, false, 0); 9893} 9894 9895SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { 9896 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 9897 MFI->setFrameAddressIsTaken(true); 9898 9899 EVT VT = Op.getValueType(); 9900 DebugLoc dl = Op.getDebugLoc(); // FIXME probably not meaningful 9901 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9902 unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP; 9903 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); 9904 while (Depth--) 9905 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, 9906 MachinePointerInfo(), 9907 false, false, false, 0); 9908 return FrameAddr; 9909} 9910 9911SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 9912 SelectionDAG &DAG) const { 9913 return DAG.getIntPtrConstant(2*TD->getPointerSize()); 9914} 9915 9916SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { 9917 MachineFunction &MF = DAG.getMachineFunction(); 9918 SDValue Chain = Op.getOperand(0); 9919 SDValue Offset = Op.getOperand(1); 9920 SDValue Handler = Op.getOperand(2); 9921 DebugLoc dl = Op.getDebugLoc(); 9922 9923 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, 9924 Subtarget->is64Bit() ? X86::RBP : X86::EBP, 9925 getPointerTy()); 9926 unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX); 9927 9928 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Frame, 9929 DAG.getIntPtrConstant(TD->getPointerSize())); 9930 StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset); 9931 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(), 9932 false, false, 0); 9933 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr); 9934 MF.getRegInfo().addLiveOut(StoreAddrReg); 9935 9936 return DAG.getNode(X86ISD::EH_RETURN, dl, 9937 MVT::Other, 9938 Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); 9939} 9940 9941SDValue X86TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 9942 SelectionDAG &DAG) const { 9943 return Op.getOperand(0); 9944} 9945 9946SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 9947 SelectionDAG &DAG) const { 9948 SDValue Root = Op.getOperand(0); 9949 SDValue Trmp = Op.getOperand(1); // trampoline 9950 SDValue FPtr = Op.getOperand(2); // nested function 9951 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 9952 DebugLoc dl = Op.getDebugLoc(); 9953 9954 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 9955 9956 if (Subtarget->is64Bit()) { 9957 SDValue OutChains[6]; 9958 9959 // Large code-model. 9960 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode. 9961 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode. 9962 9963 const unsigned char N86R10 = X86_MC::getX86RegNum(X86::R10); 9964 const unsigned char N86R11 = X86_MC::getX86RegNum(X86::R11); 9965 9966 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 9967 9968 // Load the pointer to the nested function into R11. 9969 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 9970 SDValue Addr = Trmp; 9971 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 9972 Addr, MachinePointerInfo(TrmpAddr), 9973 false, false, 0); 9974 9975 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 9976 DAG.getConstant(2, MVT::i64)); 9977 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, 9978 MachinePointerInfo(TrmpAddr, 2), 9979 false, false, 2); 9980 9981 // Load the 'nest' parameter value into R10. 9982 // R10 is specified in X86CallingConv.td 9983 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 9984 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 9985 DAG.getConstant(10, MVT::i64)); 9986 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 9987 Addr, MachinePointerInfo(TrmpAddr, 10), 9988 false, false, 0); 9989 9990 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 9991 DAG.getConstant(12, MVT::i64)); 9992 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, 9993 MachinePointerInfo(TrmpAddr, 12), 9994 false, false, 2); 9995 9996 // Jump to the nested function. 9997 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 9998 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 9999 DAG.getConstant(20, MVT::i64)); 10000 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 10001 Addr, MachinePointerInfo(TrmpAddr, 20), 10002 false, false, 0); 10003 10004 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 10005 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 10006 DAG.getConstant(22, MVT::i64)); 10007 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, 10008 MachinePointerInfo(TrmpAddr, 22), 10009 false, false, 0); 10010 10011 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6); 10012 } else { 10013 const Function *Func = 10014 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 10015 CallingConv::ID CC = Func->getCallingConv(); 10016 unsigned NestReg; 10017 10018 switch (CC) { 10019 default: 10020 llvm_unreachable("Unsupported calling convention"); 10021 case CallingConv::C: 10022 case CallingConv::X86_StdCall: { 10023 // Pass 'nest' parameter in ECX. 10024 // Must be kept in sync with X86CallingConv.td 10025 NestReg = X86::ECX; 10026 10027 // Check that ECX wasn't needed by an 'inreg' parameter. 10028 FunctionType *FTy = Func->getFunctionType(); 10029 const AttrListPtr &Attrs = Func->getAttributes(); 10030 10031 if (!Attrs.isEmpty() && !Func->isVarArg()) { 10032 unsigned InRegCount = 0; 10033 unsigned Idx = 1; 10034 10035 for (FunctionType::param_iterator I = FTy->param_begin(), 10036 E = FTy->param_end(); I != E; ++I, ++Idx) 10037 if (Attrs.paramHasAttr(Idx, Attribute::InReg)) 10038 // FIXME: should only count parameters that are lowered to integers. 10039 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32; 10040 10041 if (InRegCount > 2) { 10042 report_fatal_error("Nest register in use - reduce number of inreg" 10043 " parameters!"); 10044 } 10045 } 10046 break; 10047 } 10048 case CallingConv::X86_FastCall: 10049 case CallingConv::X86_ThisCall: 10050 case CallingConv::Fast: 10051 // Pass 'nest' parameter in EAX. 10052 // Must be kept in sync with X86CallingConv.td 10053 NestReg = X86::EAX; 10054 break; 10055 } 10056 10057 SDValue OutChains[4]; 10058 SDValue Addr, Disp; 10059 10060 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 10061 DAG.getConstant(10, MVT::i32)); 10062 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); 10063 10064 // This is storing the opcode for MOV32ri. 10065 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte. 10066 const unsigned char N86Reg = X86_MC::getX86RegNum(NestReg); 10067 OutChains[0] = DAG.getStore(Root, dl, 10068 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 10069 Trmp, MachinePointerInfo(TrmpAddr), 10070 false, false, 0); 10071 10072 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 10073 DAG.getConstant(1, MVT::i32)); 10074 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, 10075 MachinePointerInfo(TrmpAddr, 1), 10076 false, false, 1); 10077 10078 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode. 10079 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 10080 DAG.getConstant(5, MVT::i32)); 10081 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, 10082 MachinePointerInfo(TrmpAddr, 5), 10083 false, false, 1); 10084 10085 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 10086 DAG.getConstant(6, MVT::i32)); 10087 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, 10088 MachinePointerInfo(TrmpAddr, 6), 10089 false, false, 1); 10090 10091 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4); 10092 } 10093} 10094 10095SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, 10096 SelectionDAG &DAG) const { 10097 /* 10098 The rounding mode is in bits 11:10 of FPSR, and has the following 10099 settings: 10100 00 Round to nearest 10101 01 Round to -inf 10102 10 Round to +inf 10103 11 Round to 0 10104 10105 FLT_ROUNDS, on the other hand, expects the following: 10106 -1 Undefined 10107 0 Round to 0 10108 1 Round to nearest 10109 2 Round to +inf 10110 3 Round to -inf 10111 10112 To perform the conversion, we do: 10113 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 10114 */ 10115 10116 MachineFunction &MF = DAG.getMachineFunction(); 10117 const TargetMachine &TM = MF.getTarget(); 10118 const TargetFrameLowering &TFI = *TM.getFrameLowering(); 10119 unsigned StackAlignment = TFI.getStackAlignment(); 10120 EVT VT = Op.getValueType(); 10121 DebugLoc DL = Op.getDebugLoc(); 10122 10123 // Save FP Control Word to stack slot 10124 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false); 10125 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 10126 10127 10128 MachineMemOperand *MMO = 10129 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI), 10130 MachineMemOperand::MOStore, 2, 2); 10131 10132 SDValue Ops[] = { DAG.getEntryNode(), StackSlot }; 10133 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL, 10134 DAG.getVTList(MVT::Other), 10135 Ops, 2, MVT::i16, MMO); 10136 10137 // Load FP Control Word from stack slot 10138 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, 10139 MachinePointerInfo(), false, false, false, 0); 10140 10141 // Transform as necessary 10142 SDValue CWD1 = 10143 DAG.getNode(ISD::SRL, DL, MVT::i16, 10144 DAG.getNode(ISD::AND, DL, MVT::i16, 10145 CWD, DAG.getConstant(0x800, MVT::i16)), 10146 DAG.getConstant(11, MVT::i8)); 10147 SDValue CWD2 = 10148 DAG.getNode(ISD::SRL, DL, MVT::i16, 10149 DAG.getNode(ISD::AND, DL, MVT::i16, 10150 CWD, DAG.getConstant(0x400, MVT::i16)), 10151 DAG.getConstant(9, MVT::i8)); 10152 10153 SDValue RetVal = 10154 DAG.getNode(ISD::AND, DL, MVT::i16, 10155 DAG.getNode(ISD::ADD, DL, MVT::i16, 10156 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), 10157 DAG.getConstant(1, MVT::i16)), 10158 DAG.getConstant(3, MVT::i16)); 10159 10160 10161 return DAG.getNode((VT.getSizeInBits() < 16 ? 10162 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal); 10163} 10164 10165SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const { 10166 EVT VT = Op.getValueType(); 10167 EVT OpVT = VT; 10168 unsigned NumBits = VT.getSizeInBits(); 10169 DebugLoc dl = Op.getDebugLoc(); 10170 10171 Op = Op.getOperand(0); 10172 if (VT == MVT::i8) { 10173 // Zero extend to i32 since there is not an i8 bsr. 10174 OpVT = MVT::i32; 10175 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 10176 } 10177 10178 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 10179 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 10180 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); 10181 10182 // If src is zero (i.e. bsr sets ZF), returns NumBits. 10183 SDValue Ops[] = { 10184 Op, 10185 DAG.getConstant(NumBits+NumBits-1, OpVT), 10186 DAG.getConstant(X86::COND_E, MVT::i8), 10187 Op.getValue(1) 10188 }; 10189 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops)); 10190 10191 // Finally xor with NumBits-1. 10192 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 10193 10194 if (VT == MVT::i8) 10195 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 10196 return Op; 10197} 10198 10199SDValue X86TargetLowering::LowerCTLZ_ZERO_UNDEF(SDValue Op, 10200 SelectionDAG &DAG) const { 10201 EVT VT = Op.getValueType(); 10202 EVT OpVT = VT; 10203 unsigned NumBits = VT.getSizeInBits(); 10204 DebugLoc dl = Op.getDebugLoc(); 10205 10206 Op = Op.getOperand(0); 10207 if (VT == MVT::i8) { 10208 // Zero extend to i32 since there is not an i8 bsr. 10209 OpVT = MVT::i32; 10210 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 10211 } 10212 10213 // Issue a bsr (scan bits in reverse). 10214 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 10215 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); 10216 10217 // And xor with NumBits-1. 10218 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 10219 10220 if (VT == MVT::i8) 10221 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 10222 return Op; 10223} 10224 10225SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const { 10226 EVT VT = Op.getValueType(); 10227 unsigned NumBits = VT.getSizeInBits(); 10228 DebugLoc dl = Op.getDebugLoc(); 10229 Op = Op.getOperand(0); 10230 10231 // Issue a bsf (scan bits forward) which also sets EFLAGS. 10232 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 10233 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op); 10234 10235 // If src is zero (i.e. bsf sets ZF), returns NumBits. 10236 SDValue Ops[] = { 10237 Op, 10238 DAG.getConstant(NumBits, VT), 10239 DAG.getConstant(X86::COND_E, MVT::i8), 10240 Op.getValue(1) 10241 }; 10242 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops, array_lengthof(Ops)); 10243} 10244 10245// Lower256IntArith - Break a 256-bit integer operation into two new 128-bit 10246// ones, and then concatenate the result back. 10247static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) { 10248 EVT VT = Op.getValueType(); 10249 10250 assert(VT.getSizeInBits() == 256 && VT.isInteger() && 10251 "Unsupported value type for operation"); 10252 10253 unsigned NumElems = VT.getVectorNumElements(); 10254 DebugLoc dl = Op.getDebugLoc(); 10255 10256 // Extract the LHS vectors 10257 SDValue LHS = Op.getOperand(0); 10258 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl); 10259 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl); 10260 10261 // Extract the RHS vectors 10262 SDValue RHS = Op.getOperand(1); 10263 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl); 10264 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl); 10265 10266 MVT EltVT = VT.getVectorElementType().getSimpleVT(); 10267 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); 10268 10269 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, 10270 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1), 10271 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2)); 10272} 10273 10274SDValue X86TargetLowering::LowerADD(SDValue Op, SelectionDAG &DAG) const { 10275 assert(Op.getValueType().getSizeInBits() == 256 && 10276 Op.getValueType().isInteger() && 10277 "Only handle AVX 256-bit vector integer operation"); 10278 return Lower256IntArith(Op, DAG); 10279} 10280 10281SDValue X86TargetLowering::LowerSUB(SDValue Op, SelectionDAG &DAG) const { 10282 assert(Op.getValueType().getSizeInBits() == 256 && 10283 Op.getValueType().isInteger() && 10284 "Only handle AVX 256-bit vector integer operation"); 10285 return Lower256IntArith(Op, DAG); 10286} 10287 10288SDValue X86TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 10289 EVT VT = Op.getValueType(); 10290 10291 // Decompose 256-bit ops into smaller 128-bit ops. 10292 if (VT.getSizeInBits() == 256 && !Subtarget->hasAVX2()) 10293 return Lower256IntArith(Op, DAG); 10294 10295 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && 10296 "Only know how to lower V2I64/V4I64 multiply"); 10297 10298 DebugLoc dl = Op.getDebugLoc(); 10299 10300 // Ahi = psrlqi(a, 32); 10301 // Bhi = psrlqi(b, 32); 10302 // 10303 // AloBlo = pmuludq(a, b); 10304 // AloBhi = pmuludq(a, Bhi); 10305 // AhiBlo = pmuludq(Ahi, b); 10306 10307 // AloBhi = psllqi(AloBhi, 32); 10308 // AhiBlo = psllqi(AhiBlo, 32); 10309 // return AloBlo + AloBhi + AhiBlo; 10310 10311 SDValue A = Op.getOperand(0); 10312 SDValue B = Op.getOperand(1); 10313 10314 SDValue ShAmt = DAG.getConstant(32, MVT::i32); 10315 10316 SDValue Ahi = DAG.getNode(X86ISD::VSRLI, dl, VT, A, ShAmt); 10317 SDValue Bhi = DAG.getNode(X86ISD::VSRLI, dl, VT, B, ShAmt); 10318 10319 // Bit cast to 32-bit vectors for MULUDQ 10320 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 : MVT::v8i32; 10321 A = DAG.getNode(ISD::BITCAST, dl, MulVT, A); 10322 B = DAG.getNode(ISD::BITCAST, dl, MulVT, B); 10323 Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi); 10324 Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi); 10325 10326 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B); 10327 SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi); 10328 SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B); 10329 10330 AloBhi = DAG.getNode(X86ISD::VSHLI, dl, VT, AloBhi, ShAmt); 10331 AhiBlo = DAG.getNode(X86ISD::VSHLI, dl, VT, AhiBlo, ShAmt); 10332 10333 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi); 10334 return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo); 10335} 10336 10337SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { 10338 10339 EVT VT = Op.getValueType(); 10340 DebugLoc dl = Op.getDebugLoc(); 10341 SDValue R = Op.getOperand(0); 10342 SDValue Amt = Op.getOperand(1); 10343 LLVMContext *Context = DAG.getContext(); 10344 10345 if (!Subtarget->hasSSE2()) 10346 return SDValue(); 10347 10348 // Optimize shl/srl/sra with constant shift amount. 10349 if (isSplatVector(Amt.getNode())) { 10350 SDValue SclrAmt = Amt->getOperand(0); 10351 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SclrAmt)) { 10352 uint64_t ShiftAmt = C->getZExtValue(); 10353 10354 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || 10355 (Subtarget->hasAVX2() && 10356 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16))) { 10357 if (Op.getOpcode() == ISD::SHL) 10358 return DAG.getNode(X86ISD::VSHLI, dl, VT, R, 10359 DAG.getConstant(ShiftAmt, MVT::i32)); 10360 if (Op.getOpcode() == ISD::SRL) 10361 return DAG.getNode(X86ISD::VSRLI, dl, VT, R, 10362 DAG.getConstant(ShiftAmt, MVT::i32)); 10363 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64) 10364 return DAG.getNode(X86ISD::VSRAI, dl, VT, R, 10365 DAG.getConstant(ShiftAmt, MVT::i32)); 10366 } 10367 10368 if (VT == MVT::v16i8) { 10369 if (Op.getOpcode() == ISD::SHL) { 10370 // Make a large shift. 10371 SDValue SHL = DAG.getNode(X86ISD::VSHLI, dl, MVT::v8i16, R, 10372 DAG.getConstant(ShiftAmt, MVT::i32)); 10373 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL); 10374 // Zero out the rightmost bits. 10375 SmallVector<SDValue, 16> V(16, 10376 DAG.getConstant(uint8_t(-1U << ShiftAmt), 10377 MVT::i8)); 10378 return DAG.getNode(ISD::AND, dl, VT, SHL, 10379 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16)); 10380 } 10381 if (Op.getOpcode() == ISD::SRL) { 10382 // Make a large shift. 10383 SDValue SRL = DAG.getNode(X86ISD::VSRLI, dl, MVT::v8i16, R, 10384 DAG.getConstant(ShiftAmt, MVT::i32)); 10385 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL); 10386 // Zero out the leftmost bits. 10387 SmallVector<SDValue, 16> V(16, 10388 DAG.getConstant(uint8_t(-1U) >> ShiftAmt, 10389 MVT::i8)); 10390 return DAG.getNode(ISD::AND, dl, VT, SRL, 10391 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16)); 10392 } 10393 if (Op.getOpcode() == ISD::SRA) { 10394 if (ShiftAmt == 7) { 10395 // R s>> 7 === R s< 0 10396 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl); 10397 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); 10398 } 10399 10400 // R s>> a === ((R u>> a) ^ m) - m 10401 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); 10402 SmallVector<SDValue, 16> V(16, DAG.getConstant(128 >> ShiftAmt, 10403 MVT::i8)); 10404 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16); 10405 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); 10406 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); 10407 return Res; 10408 } 10409 llvm_unreachable("Unknown shift opcode."); 10410 } 10411 10412 if (Subtarget->hasAVX2() && VT == MVT::v32i8) { 10413 if (Op.getOpcode() == ISD::SHL) { 10414 // Make a large shift. 10415 SDValue SHL = DAG.getNode(X86ISD::VSHLI, dl, MVT::v16i16, R, 10416 DAG.getConstant(ShiftAmt, MVT::i32)); 10417 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL); 10418 // Zero out the rightmost bits. 10419 SmallVector<SDValue, 32> V(32, 10420 DAG.getConstant(uint8_t(-1U << ShiftAmt), 10421 MVT::i8)); 10422 return DAG.getNode(ISD::AND, dl, VT, SHL, 10423 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32)); 10424 } 10425 if (Op.getOpcode() == ISD::SRL) { 10426 // Make a large shift. 10427 SDValue SRL = DAG.getNode(X86ISD::VSRLI, dl, MVT::v16i16, R, 10428 DAG.getConstant(ShiftAmt, MVT::i32)); 10429 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL); 10430 // Zero out the leftmost bits. 10431 SmallVector<SDValue, 32> V(32, 10432 DAG.getConstant(uint8_t(-1U) >> ShiftAmt, 10433 MVT::i8)); 10434 return DAG.getNode(ISD::AND, dl, VT, SRL, 10435 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32)); 10436 } 10437 if (Op.getOpcode() == ISD::SRA) { 10438 if (ShiftAmt == 7) { 10439 // R s>> 7 === R s< 0 10440 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl); 10441 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); 10442 } 10443 10444 // R s>> a === ((R u>> a) ^ m) - m 10445 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); 10446 SmallVector<SDValue, 32> V(32, DAG.getConstant(128 >> ShiftAmt, 10447 MVT::i8)); 10448 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32); 10449 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); 10450 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); 10451 return Res; 10452 } 10453 llvm_unreachable("Unknown shift opcode."); 10454 } 10455 } 10456 } 10457 10458 // Lower SHL with variable shift amount. 10459 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { 10460 Op = DAG.getNode(X86ISD::VSHLI, dl, VT, Op.getOperand(1), 10461 DAG.getConstant(23, MVT::i32)); 10462 10463 const uint32_t CV[] = { 0x3f800000U, 0x3f800000U, 0x3f800000U, 0x3f800000U}; 10464 Constant *C = ConstantDataVector::get(*Context, CV); 10465 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 10466 SDValue Addend = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 10467 MachinePointerInfo::getConstantPool(), 10468 false, false, false, 16); 10469 10470 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Addend); 10471 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); 10472 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); 10473 return DAG.getNode(ISD::MUL, dl, VT, Op, R); 10474 } 10475 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { 10476 assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq."); 10477 10478 // a = a << 5; 10479 Op = DAG.getNode(X86ISD::VSHLI, dl, MVT::v8i16, Op.getOperand(1), 10480 DAG.getConstant(5, MVT::i32)); 10481 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op); 10482 10483 // Turn 'a' into a mask suitable for VSELECT 10484 SDValue VSelM = DAG.getConstant(0x80, VT); 10485 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); 10486 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM); 10487 10488 SDValue CM1 = DAG.getConstant(0x0f, VT); 10489 SDValue CM2 = DAG.getConstant(0x3f, VT); 10490 10491 // r = VSELECT(r, psllw(r & (char16)15, 4), a); 10492 SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1); 10493 M = getTargetVShiftNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 10494 DAG.getConstant(4, MVT::i32), DAG); 10495 M = DAG.getNode(ISD::BITCAST, dl, VT, M); 10496 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R); 10497 10498 // a += a 10499 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); 10500 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); 10501 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM); 10502 10503 // r = VSELECT(r, psllw(r & (char16)63, 2), a); 10504 M = DAG.getNode(ISD::AND, dl, VT, R, CM2); 10505 M = getTargetVShiftNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 10506 DAG.getConstant(2, MVT::i32), DAG); 10507 M = DAG.getNode(ISD::BITCAST, dl, VT, M); 10508 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R); 10509 10510 // a += a 10511 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); 10512 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); 10513 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM); 10514 10515 // return VSELECT(r, r+r, a); 10516 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, 10517 DAG.getNode(ISD::ADD, dl, VT, R, R), R); 10518 return R; 10519 } 10520 10521 // Decompose 256-bit shifts into smaller 128-bit shifts. 10522 if (VT.getSizeInBits() == 256) { 10523 unsigned NumElems = VT.getVectorNumElements(); 10524 MVT EltVT = VT.getVectorElementType().getSimpleVT(); 10525 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); 10526 10527 // Extract the two vectors 10528 SDValue V1 = Extract128BitVector(R, 0, DAG, dl); 10529 SDValue V2 = Extract128BitVector(R, NumElems/2, DAG, dl); 10530 10531 // Recreate the shift amount vectors 10532 SDValue Amt1, Amt2; 10533 if (Amt.getOpcode() == ISD::BUILD_VECTOR) { 10534 // Constant shift amount 10535 SmallVector<SDValue, 4> Amt1Csts; 10536 SmallVector<SDValue, 4> Amt2Csts; 10537 for (unsigned i = 0; i != NumElems/2; ++i) 10538 Amt1Csts.push_back(Amt->getOperand(i)); 10539 for (unsigned i = NumElems/2; i != NumElems; ++i) 10540 Amt2Csts.push_back(Amt->getOperand(i)); 10541 10542 Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, 10543 &Amt1Csts[0], NumElems/2); 10544 Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, 10545 &Amt2Csts[0], NumElems/2); 10546 } else { 10547 // Variable shift amount 10548 Amt1 = Extract128BitVector(Amt, 0, DAG, dl); 10549 Amt2 = Extract128BitVector(Amt, NumElems/2, DAG, dl); 10550 } 10551 10552 // Issue new vector shifts for the smaller types 10553 V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1); 10554 V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2); 10555 10556 // Concatenate the result back 10557 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2); 10558 } 10559 10560 return SDValue(); 10561} 10562 10563SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const { 10564 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus 10565 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering 10566 // looks for this combo and may remove the "setcc" instruction if the "setcc" 10567 // has only one use. 10568 SDNode *N = Op.getNode(); 10569 SDValue LHS = N->getOperand(0); 10570 SDValue RHS = N->getOperand(1); 10571 unsigned BaseOp = 0; 10572 unsigned Cond = 0; 10573 DebugLoc DL = Op.getDebugLoc(); 10574 switch (Op.getOpcode()) { 10575 default: llvm_unreachable("Unknown ovf instruction!"); 10576 case ISD::SADDO: 10577 // A subtract of one will be selected as a INC. Note that INC doesn't 10578 // set CF, so we can't do this for UADDO. 10579 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) 10580 if (C->isOne()) { 10581 BaseOp = X86ISD::INC; 10582 Cond = X86::COND_O; 10583 break; 10584 } 10585 BaseOp = X86ISD::ADD; 10586 Cond = X86::COND_O; 10587 break; 10588 case ISD::UADDO: 10589 BaseOp = X86ISD::ADD; 10590 Cond = X86::COND_B; 10591 break; 10592 case ISD::SSUBO: 10593 // A subtract of one will be selected as a DEC. Note that DEC doesn't 10594 // set CF, so we can't do this for USUBO. 10595 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) 10596 if (C->isOne()) { 10597 BaseOp = X86ISD::DEC; 10598 Cond = X86::COND_O; 10599 break; 10600 } 10601 BaseOp = X86ISD::SUB; 10602 Cond = X86::COND_O; 10603 break; 10604 case ISD::USUBO: 10605 BaseOp = X86ISD::SUB; 10606 Cond = X86::COND_B; 10607 break; 10608 case ISD::SMULO: 10609 BaseOp = X86ISD::SMUL; 10610 Cond = X86::COND_O; 10611 break; 10612 case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs 10613 SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0), 10614 MVT::i32); 10615 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS); 10616 10617 SDValue SetCC = 10618 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 10619 DAG.getConstant(X86::COND_O, MVT::i32), 10620 SDValue(Sum.getNode(), 2)); 10621 10622 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); 10623 } 10624 } 10625 10626 // Also sets EFLAGS. 10627 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); 10628 SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); 10629 10630 SDValue SetCC = 10631 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1), 10632 DAG.getConstant(Cond, MVT::i32), 10633 SDValue(Sum.getNode(), 1)); 10634 10635 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); 10636} 10637 10638SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 10639 SelectionDAG &DAG) const { 10640 DebugLoc dl = Op.getDebugLoc(); 10641 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 10642 EVT VT = Op.getValueType(); 10643 10644 if (!Subtarget->hasSSE2() || !VT.isVector()) 10645 return SDValue(); 10646 10647 unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 10648 ExtraVT.getScalarType().getSizeInBits(); 10649 SDValue ShAmt = DAG.getConstant(BitsDiff, MVT::i32); 10650 10651 switch (VT.getSimpleVT().SimpleTy) { 10652 default: return SDValue(); 10653 case MVT::v8i32: 10654 case MVT::v16i16: 10655 if (!Subtarget->hasAVX()) 10656 return SDValue(); 10657 if (!Subtarget->hasAVX2()) { 10658 // needs to be split 10659 unsigned NumElems = VT.getVectorNumElements(); 10660 10661 // Extract the LHS vectors 10662 SDValue LHS = Op.getOperand(0); 10663 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl); 10664 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl); 10665 10666 MVT EltVT = VT.getVectorElementType().getSimpleVT(); 10667 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); 10668 10669 EVT ExtraEltVT = ExtraVT.getVectorElementType(); 10670 unsigned ExtraNumElems = ExtraVT.getVectorNumElements(); 10671 ExtraVT = EVT::getVectorVT(*DAG.getContext(), ExtraEltVT, 10672 ExtraNumElems/2); 10673 SDValue Extra = DAG.getValueType(ExtraVT); 10674 10675 LHS1 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, Extra); 10676 LHS2 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, Extra); 10677 10678 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);; 10679 } 10680 // fall through 10681 case MVT::v4i32: 10682 case MVT::v8i16: { 10683 SDValue Tmp1 = getTargetVShiftNode(X86ISD::VSHLI, dl, VT, 10684 Op.getOperand(0), ShAmt, DAG); 10685 return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, Tmp1, ShAmt, DAG); 10686 } 10687 } 10688} 10689 10690 10691SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{ 10692 DebugLoc dl = Op.getDebugLoc(); 10693 10694 // Go ahead and emit the fence on x86-64 even if we asked for no-sse2. 10695 // There isn't any reason to disable it if the target processor supports it. 10696 if (!Subtarget->hasSSE2() && !Subtarget->is64Bit()) { 10697 SDValue Chain = Op.getOperand(0); 10698 SDValue Zero = DAG.getConstant(0, MVT::i32); 10699 SDValue Ops[] = { 10700 DAG.getRegister(X86::ESP, MVT::i32), // Base 10701 DAG.getTargetConstant(1, MVT::i8), // Scale 10702 DAG.getRegister(0, MVT::i32), // Index 10703 DAG.getTargetConstant(0, MVT::i32), // Disp 10704 DAG.getRegister(0, MVT::i32), // Segment. 10705 Zero, 10706 Chain 10707 }; 10708 SDNode *Res = 10709 DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops, 10710 array_lengthof(Ops)); 10711 return SDValue(Res, 0); 10712 } 10713 10714 unsigned isDev = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue(); 10715 if (!isDev) 10716 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); 10717 10718 unsigned Op1 = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 10719 unsigned Op2 = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 10720 unsigned Op3 = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 10721 unsigned Op4 = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 10722 10723 // def : Pat<(membarrier (i8 0), (i8 0), (i8 0), (i8 1), (i8 1)), (SFENCE)>; 10724 if (!Op1 && !Op2 && !Op3 && Op4) 10725 return DAG.getNode(X86ISD::SFENCE, dl, MVT::Other, Op.getOperand(0)); 10726 10727 // def : Pat<(membarrier (i8 1), (i8 0), (i8 0), (i8 0), (i8 1)), (LFENCE)>; 10728 if (Op1 && !Op2 && !Op3 && !Op4) 10729 return DAG.getNode(X86ISD::LFENCE, dl, MVT::Other, Op.getOperand(0)); 10730 10731 // def : Pat<(membarrier (i8 imm), (i8 imm), (i8 imm), (i8 imm), (i8 1)), 10732 // (MFENCE)>; 10733 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); 10734} 10735 10736SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op, 10737 SelectionDAG &DAG) const { 10738 DebugLoc dl = Op.getDebugLoc(); 10739 AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>( 10740 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()); 10741 SynchronizationScope FenceScope = static_cast<SynchronizationScope>( 10742 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue()); 10743 10744 // The only fence that needs an instruction is a sequentially-consistent 10745 // cross-thread fence. 10746 if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) { 10747 // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for 10748 // no-sse2). There isn't any reason to disable it if the target processor 10749 // supports it. 10750 if (Subtarget->hasSSE2() || Subtarget->is64Bit()) 10751 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); 10752 10753 SDValue Chain = Op.getOperand(0); 10754 SDValue Zero = DAG.getConstant(0, MVT::i32); 10755 SDValue Ops[] = { 10756 DAG.getRegister(X86::ESP, MVT::i32), // Base 10757 DAG.getTargetConstant(1, MVT::i8), // Scale 10758 DAG.getRegister(0, MVT::i32), // Index 10759 DAG.getTargetConstant(0, MVT::i32), // Disp 10760 DAG.getRegister(0, MVT::i32), // Segment. 10761 Zero, 10762 Chain 10763 }; 10764 SDNode *Res = 10765 DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops, 10766 array_lengthof(Ops)); 10767 return SDValue(Res, 0); 10768 } 10769 10770 // MEMBARRIER is a compiler barrier; it codegens to a no-op. 10771 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); 10772} 10773 10774 10775SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const { 10776 EVT T = Op.getValueType(); 10777 DebugLoc DL = Op.getDebugLoc(); 10778 unsigned Reg = 0; 10779 unsigned size = 0; 10780 switch(T.getSimpleVT().SimpleTy) { 10781 default: llvm_unreachable("Invalid value type!"); 10782 case MVT::i8: Reg = X86::AL; size = 1; break; 10783 case MVT::i16: Reg = X86::AX; size = 2; break; 10784 case MVT::i32: Reg = X86::EAX; size = 4; break; 10785 case MVT::i64: 10786 assert(Subtarget->is64Bit() && "Node not type legal!"); 10787 Reg = X86::RAX; size = 8; 10788 break; 10789 } 10790 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg, 10791 Op.getOperand(2), SDValue()); 10792 SDValue Ops[] = { cpIn.getValue(0), 10793 Op.getOperand(1), 10794 Op.getOperand(3), 10795 DAG.getTargetConstant(size, MVT::i8), 10796 cpIn.getValue(1) }; 10797 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 10798 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand(); 10799 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, 10800 Ops, 5, T, MMO); 10801 SDValue cpOut = 10802 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); 10803 return cpOut; 10804} 10805 10806SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op, 10807 SelectionDAG &DAG) const { 10808 assert(Subtarget->is64Bit() && "Result not type legalized?"); 10809 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 10810 SDValue TheChain = Op.getOperand(0); 10811 DebugLoc dl = Op.getDebugLoc(); 10812 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 10813 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1)); 10814 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64, 10815 rax.getValue(2)); 10816 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx, 10817 DAG.getConstant(32, MVT::i8)); 10818 SDValue Ops[] = { 10819 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp), 10820 rdx.getValue(1) 10821 }; 10822 return DAG.getMergeValues(Ops, 2, dl); 10823} 10824 10825SDValue X86TargetLowering::LowerBITCAST(SDValue Op, 10826 SelectionDAG &DAG) const { 10827 EVT SrcVT = Op.getOperand(0).getValueType(); 10828 EVT DstVT = Op.getValueType(); 10829 assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() && 10830 Subtarget->hasMMX() && "Unexpected custom BITCAST"); 10831 assert((DstVT == MVT::i64 || 10832 (DstVT.isVector() && DstVT.getSizeInBits()==64)) && 10833 "Unexpected custom BITCAST"); 10834 // i64 <=> MMX conversions are Legal. 10835 if (SrcVT==MVT::i64 && DstVT.isVector()) 10836 return Op; 10837 if (DstVT==MVT::i64 && SrcVT.isVector()) 10838 return Op; 10839 // MMX <=> MMX conversions are Legal. 10840 if (SrcVT.isVector() && DstVT.isVector()) 10841 return Op; 10842 // All other conversions need to be expanded. 10843 return SDValue(); 10844} 10845 10846SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const { 10847 SDNode *Node = Op.getNode(); 10848 DebugLoc dl = Node->getDebugLoc(); 10849 EVT T = Node->getValueType(0); 10850 SDValue negOp = DAG.getNode(ISD::SUB, dl, T, 10851 DAG.getConstant(0, T), Node->getOperand(2)); 10852 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, 10853 cast<AtomicSDNode>(Node)->getMemoryVT(), 10854 Node->getOperand(0), 10855 Node->getOperand(1), negOp, 10856 cast<AtomicSDNode>(Node)->getSrcValue(), 10857 cast<AtomicSDNode>(Node)->getAlignment(), 10858 cast<AtomicSDNode>(Node)->getOrdering(), 10859 cast<AtomicSDNode>(Node)->getSynchScope()); 10860} 10861 10862static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) { 10863 SDNode *Node = Op.getNode(); 10864 DebugLoc dl = Node->getDebugLoc(); 10865 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT(); 10866 10867 // Convert seq_cst store -> xchg 10868 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b) 10869 // FIXME: On 32-bit, store -> fist or movq would be more efficient 10870 // (The only way to get a 16-byte store is cmpxchg16b) 10871 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment. 10872 if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent || 10873 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) { 10874 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 10875 cast<AtomicSDNode>(Node)->getMemoryVT(), 10876 Node->getOperand(0), 10877 Node->getOperand(1), Node->getOperand(2), 10878 cast<AtomicSDNode>(Node)->getMemOperand(), 10879 cast<AtomicSDNode>(Node)->getOrdering(), 10880 cast<AtomicSDNode>(Node)->getSynchScope()); 10881 return Swap.getValue(1); 10882 } 10883 // Other atomic stores have a simple pattern. 10884 return Op; 10885} 10886 10887static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) { 10888 EVT VT = Op.getNode()->getValueType(0); 10889 10890 // Let legalize expand this if it isn't a legal type yet. 10891 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 10892 return SDValue(); 10893 10894 SDVTList VTs = DAG.getVTList(VT, MVT::i32); 10895 10896 unsigned Opc; 10897 bool ExtraOp = false; 10898 switch (Op.getOpcode()) { 10899 default: llvm_unreachable("Invalid code"); 10900 case ISD::ADDC: Opc = X86ISD::ADD; break; 10901 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break; 10902 case ISD::SUBC: Opc = X86ISD::SUB; break; 10903 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break; 10904 } 10905 10906 if (!ExtraOp) 10907 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), 10908 Op.getOperand(1)); 10909 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), 10910 Op.getOperand(1), Op.getOperand(2)); 10911} 10912 10913/// LowerOperation - Provide custom lowering hooks for some operations. 10914/// 10915SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 10916 switch (Op.getOpcode()) { 10917 default: llvm_unreachable("Should not custom lower this!"); 10918 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG); 10919 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op,DAG); 10920 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op,DAG); 10921 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 10922 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); 10923 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG); 10924 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 10925 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 10926 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 10927 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 10928 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 10929 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 10930 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); 10931 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 10932 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 10933 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 10934 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 10935 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 10936 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 10937 case ISD::SHL_PARTS: 10938 case ISD::SRA_PARTS: 10939 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG); 10940 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 10941 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 10942 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 10943 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); 10944 case ISD::FABS: return LowerFABS(Op, DAG); 10945 case ISD::FNEG: return LowerFNEG(Op, DAG); 10946 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 10947 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG); 10948 case ISD::SETCC: return LowerSETCC(Op, DAG); 10949 case ISD::SELECT: return LowerSELECT(Op, DAG); 10950 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 10951 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 10952 case ISD::VASTART: return LowerVASTART(Op, DAG); 10953 case ISD::VAARG: return LowerVAARG(Op, DAG); 10954 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 10955 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 10956 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 10957 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 10958 case ISD::FRAME_TO_ARGS_OFFSET: 10959 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 10960 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 10961 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 10962 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 10963 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 10964 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 10965 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 10966 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ_ZERO_UNDEF(Op, DAG); 10967 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 10968 case ISD::MUL: return LowerMUL(Op, DAG); 10969 case ISD::SRA: 10970 case ISD::SRL: 10971 case ISD::SHL: return LowerShift(Op, DAG); 10972 case ISD::SADDO: 10973 case ISD::UADDO: 10974 case ISD::SSUBO: 10975 case ISD::USUBO: 10976 case ISD::SMULO: 10977 case ISD::UMULO: return LowerXALUO(Op, DAG); 10978 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); 10979 case ISD::BITCAST: return LowerBITCAST(Op, DAG); 10980 case ISD::ADDC: 10981 case ISD::ADDE: 10982 case ISD::SUBC: 10983 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); 10984 case ISD::ADD: return LowerADD(Op, DAG); 10985 case ISD::SUB: return LowerSUB(Op, DAG); 10986 } 10987} 10988 10989static void ReplaceATOMIC_LOAD(SDNode *Node, 10990 SmallVectorImpl<SDValue> &Results, 10991 SelectionDAG &DAG) { 10992 DebugLoc dl = Node->getDebugLoc(); 10993 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT(); 10994 10995 // Convert wide load -> cmpxchg8b/cmpxchg16b 10996 // FIXME: On 32-bit, load -> fild or movq would be more efficient 10997 // (The only way to get a 16-byte load is cmpxchg16b) 10998 // FIXME: 16-byte ATOMIC_CMP_SWAP isn't actually hooked up at the moment. 10999 SDValue Zero = DAG.getConstant(0, VT); 11000 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, 11001 Node->getOperand(0), 11002 Node->getOperand(1), Zero, Zero, 11003 cast<AtomicSDNode>(Node)->getMemOperand(), 11004 cast<AtomicSDNode>(Node)->getOrdering(), 11005 cast<AtomicSDNode>(Node)->getSynchScope()); 11006 Results.push_back(Swap.getValue(0)); 11007 Results.push_back(Swap.getValue(1)); 11008} 11009 11010void X86TargetLowering:: 11011ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results, 11012 SelectionDAG &DAG, unsigned NewOp) const { 11013 DebugLoc dl = Node->getDebugLoc(); 11014 assert (Node->getValueType(0) == MVT::i64 && 11015 "Only know how to expand i64 atomics"); 11016 11017 SDValue Chain = Node->getOperand(0); 11018 SDValue In1 = Node->getOperand(1); 11019 SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 11020 Node->getOperand(2), DAG.getIntPtrConstant(0)); 11021 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 11022 Node->getOperand(2), DAG.getIntPtrConstant(1)); 11023 SDValue Ops[] = { Chain, In1, In2L, In2H }; 11024 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 11025 SDValue Result = 11026 DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, 11027 cast<MemSDNode>(Node)->getMemOperand()); 11028 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; 11029 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); 11030 Results.push_back(Result.getValue(2)); 11031} 11032 11033/// ReplaceNodeResults - Replace a node with an illegal result type 11034/// with a new node built out of custom code. 11035void X86TargetLowering::ReplaceNodeResults(SDNode *N, 11036 SmallVectorImpl<SDValue>&Results, 11037 SelectionDAG &DAG) const { 11038 DebugLoc dl = N->getDebugLoc(); 11039 switch (N->getOpcode()) { 11040 default: 11041 llvm_unreachable("Do not know how to custom type legalize this operation!"); 11042 case ISD::SIGN_EXTEND_INREG: 11043 case ISD::ADDC: 11044 case ISD::ADDE: 11045 case ISD::SUBC: 11046 case ISD::SUBE: 11047 // We don't want to expand or promote these. 11048 return; 11049 case ISD::FP_TO_SINT: 11050 case ISD::FP_TO_UINT: { 11051 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; 11052 11053 if (!IsSigned && !isIntegerTypeFTOL(SDValue(N, 0).getValueType())) 11054 return; 11055 11056 std::pair<SDValue,SDValue> Vals = 11057 FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, /*IsReplace=*/ true); 11058 SDValue FIST = Vals.first, StackSlot = Vals.second; 11059 if (FIST.getNode() != 0) { 11060 EVT VT = N->getValueType(0); 11061 // Return a load from the stack slot. 11062 if (StackSlot.getNode() != 0) 11063 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, 11064 MachinePointerInfo(), 11065 false, false, false, 0)); 11066 else 11067 Results.push_back(FIST); 11068 } 11069 return; 11070 } 11071 case ISD::READCYCLECOUNTER: { 11072 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 11073 SDValue TheChain = N->getOperand(0); 11074 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 11075 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32, 11076 rd.getValue(1)); 11077 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32, 11078 eax.getValue(2)); 11079 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 11080 SDValue Ops[] = { eax, edx }; 11081 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2)); 11082 Results.push_back(edx.getValue(1)); 11083 return; 11084 } 11085 case ISD::ATOMIC_CMP_SWAP: { 11086 EVT T = N->getValueType(0); 11087 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"); 11088 bool Regs64bit = T == MVT::i128; 11089 EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32; 11090 SDValue cpInL, cpInH; 11091 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), 11092 DAG.getConstant(0, HalfT)); 11093 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), 11094 DAG.getConstant(1, HalfT)); 11095 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, 11096 Regs64bit ? X86::RAX : X86::EAX, 11097 cpInL, SDValue()); 11098 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, 11099 Regs64bit ? X86::RDX : X86::EDX, 11100 cpInH, cpInL.getValue(1)); 11101 SDValue swapInL, swapInH; 11102 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), 11103 DAG.getConstant(0, HalfT)); 11104 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), 11105 DAG.getConstant(1, HalfT)); 11106 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, 11107 Regs64bit ? X86::RBX : X86::EBX, 11108 swapInL, cpInH.getValue(1)); 11109 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, 11110 Regs64bit ? X86::RCX : X86::ECX, 11111 swapInH, swapInL.getValue(1)); 11112 SDValue Ops[] = { swapInH.getValue(0), 11113 N->getOperand(1), 11114 swapInH.getValue(1) }; 11115 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); 11116 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand(); 11117 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG : 11118 X86ISD::LCMPXCHG8_DAG; 11119 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, 11120 Ops, 3, T, MMO); 11121 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, 11122 Regs64bit ? X86::RAX : X86::EAX, 11123 HalfT, Result.getValue(1)); 11124 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, 11125 Regs64bit ? X86::RDX : X86::EDX, 11126 HalfT, cpOutL.getValue(2)); 11127 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 11128 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF, 2)); 11129 Results.push_back(cpOutH.getValue(1)); 11130 return; 11131 } 11132 case ISD::ATOMIC_LOAD_ADD: 11133 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG); 11134 return; 11135 case ISD::ATOMIC_LOAD_AND: 11136 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG); 11137 return; 11138 case ISD::ATOMIC_LOAD_NAND: 11139 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG); 11140 return; 11141 case ISD::ATOMIC_LOAD_OR: 11142 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG); 11143 return; 11144 case ISD::ATOMIC_LOAD_SUB: 11145 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG); 11146 return; 11147 case ISD::ATOMIC_LOAD_XOR: 11148 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG); 11149 return; 11150 case ISD::ATOMIC_SWAP: 11151 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG); 11152 return; 11153 case ISD::ATOMIC_LOAD: 11154 ReplaceATOMIC_LOAD(N, Results, DAG); 11155 } 11156} 11157 11158const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 11159 switch (Opcode) { 11160 default: return NULL; 11161 case X86ISD::BSF: return "X86ISD::BSF"; 11162 case X86ISD::BSR: return "X86ISD::BSR"; 11163 case X86ISD::SHLD: return "X86ISD::SHLD"; 11164 case X86ISD::SHRD: return "X86ISD::SHRD"; 11165 case X86ISD::FAND: return "X86ISD::FAND"; 11166 case X86ISD::FOR: return "X86ISD::FOR"; 11167 case X86ISD::FXOR: return "X86ISD::FXOR"; 11168 case X86ISD::FSRL: return "X86ISD::FSRL"; 11169 case X86ISD::FILD: return "X86ISD::FILD"; 11170 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 11171 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 11172 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 11173 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 11174 case X86ISD::FLD: return "X86ISD::FLD"; 11175 case X86ISD::FST: return "X86ISD::FST"; 11176 case X86ISD::CALL: return "X86ISD::CALL"; 11177 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 11178 case X86ISD::BT: return "X86ISD::BT"; 11179 case X86ISD::CMP: return "X86ISD::CMP"; 11180 case X86ISD::COMI: return "X86ISD::COMI"; 11181 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 11182 case X86ISD::SETCC: return "X86ISD::SETCC"; 11183 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY"; 11184 case X86ISD::FSETCCsd: return "X86ISD::FSETCCsd"; 11185 case X86ISD::FSETCCss: return "X86ISD::FSETCCss"; 11186 case X86ISD::CMOV: return "X86ISD::CMOV"; 11187 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 11188 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 11189 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 11190 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 11191 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 11192 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 11193 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP"; 11194 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 11195 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 11196 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 11197 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 11198 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 11199 case X86ISD::PSHUFB: return "X86ISD::PSHUFB"; 11200 case X86ISD::ANDNP: return "X86ISD::ANDNP"; 11201 case X86ISD::PSIGN: return "X86ISD::PSIGN"; 11202 case X86ISD::BLENDV: return "X86ISD::BLENDV"; 11203 case X86ISD::BLENDPW: return "X86ISD::BLENDPW"; 11204 case X86ISD::BLENDPS: return "X86ISD::BLENDPS"; 11205 case X86ISD::BLENDPD: return "X86ISD::BLENDPD"; 11206 case X86ISD::HADD: return "X86ISD::HADD"; 11207 case X86ISD::HSUB: return "X86ISD::HSUB"; 11208 case X86ISD::FHADD: return "X86ISD::FHADD"; 11209 case X86ISD::FHSUB: return "X86ISD::FHSUB"; 11210 case X86ISD::FMAX: return "X86ISD::FMAX"; 11211 case X86ISD::FMIN: return "X86ISD::FMIN"; 11212 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 11213 case X86ISD::FRCP: return "X86ISD::FRCP"; 11214 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 11215 case X86ISD::TLSCALL: return "X86ISD::TLSCALL"; 11216 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 11217 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 11218 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 11219 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r"; 11220 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 11221 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 11222 case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG"; 11223 case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG"; 11224 case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG"; 11225 case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG"; 11226 case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG"; 11227 case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG"; 11228 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 11229 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 11230 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ"; 11231 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ"; 11232 case X86ISD::VSHL: return "X86ISD::VSHL"; 11233 case X86ISD::VSRL: return "X86ISD::VSRL"; 11234 case X86ISD::VSRA: return "X86ISD::VSRA"; 11235 case X86ISD::VSHLI: return "X86ISD::VSHLI"; 11236 case X86ISD::VSRLI: return "X86ISD::VSRLI"; 11237 case X86ISD::VSRAI: return "X86ISD::VSRAI"; 11238 case X86ISD::CMPP: return "X86ISD::CMPP"; 11239 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ"; 11240 case X86ISD::PCMPGT: return "X86ISD::PCMPGT"; 11241 case X86ISD::ADD: return "X86ISD::ADD"; 11242 case X86ISD::SUB: return "X86ISD::SUB"; 11243 case X86ISD::ADC: return "X86ISD::ADC"; 11244 case X86ISD::SBB: return "X86ISD::SBB"; 11245 case X86ISD::SMUL: return "X86ISD::SMUL"; 11246 case X86ISD::UMUL: return "X86ISD::UMUL"; 11247 case X86ISD::INC: return "X86ISD::INC"; 11248 case X86ISD::DEC: return "X86ISD::DEC"; 11249 case X86ISD::OR: return "X86ISD::OR"; 11250 case X86ISD::XOR: return "X86ISD::XOR"; 11251 case X86ISD::AND: return "X86ISD::AND"; 11252 case X86ISD::ANDN: return "X86ISD::ANDN"; 11253 case X86ISD::BLSI: return "X86ISD::BLSI"; 11254 case X86ISD::BLSMSK: return "X86ISD::BLSMSK"; 11255 case X86ISD::BLSR: return "X86ISD::BLSR"; 11256 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM"; 11257 case X86ISD::PTEST: return "X86ISD::PTEST"; 11258 case X86ISD::TESTP: return "X86ISD::TESTP"; 11259 case X86ISD::PALIGN: return "X86ISD::PALIGN"; 11260 case X86ISD::PSHUFD: return "X86ISD::PSHUFD"; 11261 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW"; 11262 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW"; 11263 case X86ISD::SHUFP: return "X86ISD::SHUFP"; 11264 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS"; 11265 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD"; 11266 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS"; 11267 case X86ISD::MOVLPS: return "X86ISD::MOVLPS"; 11268 case X86ISD::MOVLPD: return "X86ISD::MOVLPD"; 11269 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP"; 11270 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP"; 11271 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP"; 11272 case X86ISD::MOVSD: return "X86ISD::MOVSD"; 11273 case X86ISD::MOVSS: return "X86ISD::MOVSS"; 11274 case X86ISD::UNPCKL: return "X86ISD::UNPCKL"; 11275 case X86ISD::UNPCKH: return "X86ISD::UNPCKH"; 11276 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST"; 11277 case X86ISD::VPERMILP: return "X86ISD::VPERMILP"; 11278 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128"; 11279 case X86ISD::VPERMV: return "X86ISD::VPERMV"; 11280 case X86ISD::VPERMI: return "X86ISD::VPERMI"; 11281 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ"; 11282 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS"; 11283 case X86ISD::VAARG_64: return "X86ISD::VAARG_64"; 11284 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA"; 11285 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER"; 11286 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA"; 11287 case X86ISD::WIN_FTOL: return "X86ISD::WIN_FTOL"; 11288 case X86ISD::SAHF: return "X86ISD::SAHF"; 11289 } 11290} 11291 11292// isLegalAddressingMode - Return true if the addressing mode represented 11293// by AM is legal for this target, for a load/store of the specified type. 11294bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 11295 Type *Ty) const { 11296 // X86 supports extremely general addressing modes. 11297 CodeModel::Model M = getTargetMachine().getCodeModel(); 11298 Reloc::Model R = getTargetMachine().getRelocationModel(); 11299 11300 // X86 allows a sign-extended 32-bit immediate field as a displacement. 11301 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL)) 11302 return false; 11303 11304 if (AM.BaseGV) { 11305 unsigned GVFlags = 11306 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine()); 11307 11308 // If a reference to this global requires an extra load, we can't fold it. 11309 if (isGlobalStubReference(GVFlags)) 11310 return false; 11311 11312 // If BaseGV requires a register for the PIC base, we cannot also have a 11313 // BaseReg specified. 11314 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags)) 11315 return false; 11316 11317 // If lower 4G is not available, then we must use rip-relative addressing. 11318 if ((M != CodeModel::Small || R != Reloc::Static) && 11319 Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1)) 11320 return false; 11321 } 11322 11323 switch (AM.Scale) { 11324 case 0: 11325 case 1: 11326 case 2: 11327 case 4: 11328 case 8: 11329 // These scales always work. 11330 break; 11331 case 3: 11332 case 5: 11333 case 9: 11334 // These scales are formed with basereg+scalereg. Only accept if there is 11335 // no basereg yet. 11336 if (AM.HasBaseReg) 11337 return false; 11338 break; 11339 default: // Other stuff never works. 11340 return false; 11341 } 11342 11343 return true; 11344} 11345 11346 11347bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 11348 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 11349 return false; 11350 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 11351 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 11352 if (NumBits1 <= NumBits2) 11353 return false; 11354 return true; 11355} 11356 11357bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 11358 if (!VT1.isInteger() || !VT2.isInteger()) 11359 return false; 11360 unsigned NumBits1 = VT1.getSizeInBits(); 11361 unsigned NumBits2 = VT2.getSizeInBits(); 11362 if (NumBits1 <= NumBits2) 11363 return false; 11364 return true; 11365} 11366 11367bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const { 11368 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 11369 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit(); 11370} 11371 11372bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const { 11373 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 11374 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); 11375} 11376 11377bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const { 11378 // i16 instructions are longer (0x66 prefix) and potentially slower. 11379 return !(VT1 == MVT::i32 && VT2 == MVT::i16); 11380} 11381 11382/// isShuffleMaskLegal - Targets can use this to indicate that they only 11383/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 11384/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 11385/// are assumed to be legal. 11386bool 11387X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M, 11388 EVT VT) const { 11389 // Very little shuffling can be done for 64-bit vectors right now. 11390 if (VT.getSizeInBits() == 64) 11391 return false; 11392 11393 // FIXME: pshufb, blends, shifts. 11394 return (VT.getVectorNumElements() == 2 || 11395 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || 11396 isMOVLMask(M, VT) || 11397 isSHUFPMask(M, VT, Subtarget->hasAVX()) || 11398 isPSHUFDMask(M, VT) || 11399 isPSHUFHWMask(M, VT, Subtarget->hasAVX2()) || 11400 isPSHUFLWMask(M, VT, Subtarget->hasAVX2()) || 11401 isPALIGNRMask(M, VT, Subtarget) || 11402 isUNPCKLMask(M, VT, Subtarget->hasAVX2()) || 11403 isUNPCKHMask(M, VT, Subtarget->hasAVX2()) || 11404 isUNPCKL_v_undef_Mask(M, VT, Subtarget->hasAVX2()) || 11405 isUNPCKH_v_undef_Mask(M, VT, Subtarget->hasAVX2())); 11406} 11407 11408bool 11409X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask, 11410 EVT VT) const { 11411 unsigned NumElts = VT.getVectorNumElements(); 11412 // FIXME: This collection of masks seems suspect. 11413 if (NumElts == 2) 11414 return true; 11415 if (NumElts == 4 && VT.getSizeInBits() == 128) { 11416 return (isMOVLMask(Mask, VT) || 11417 isCommutedMOVLMask(Mask, VT, true) || 11418 isSHUFPMask(Mask, VT, Subtarget->hasAVX()) || 11419 isSHUFPMask(Mask, VT, Subtarget->hasAVX(), /* Commuted */ true)); 11420 } 11421 return false; 11422} 11423 11424//===----------------------------------------------------------------------===// 11425// X86 Scheduler Hooks 11426//===----------------------------------------------------------------------===// 11427 11428// private utility function 11429MachineBasicBlock * 11430X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 11431 MachineBasicBlock *MBB, 11432 unsigned regOpc, 11433 unsigned immOpc, 11434 unsigned LoadOpc, 11435 unsigned CXchgOpc, 11436 unsigned notOpc, 11437 unsigned EAXreg, 11438 const TargetRegisterClass *RC, 11439 bool Invert) const { 11440 // For the atomic bitwise operator, we generate 11441 // thisMBB: 11442 // newMBB: 11443 // ld t1 = [bitinstr.addr] 11444 // op t2 = t1, [bitinstr.val] 11445 // not t3 = t2 (if Invert) 11446 // mov EAX = t1 11447 // lcs dest = [bitinstr.addr], t3 [EAX is implicit] 11448 // bz newMBB 11449 // fallthrough -->nextMBB 11450 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11451 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 11452 MachineFunction::iterator MBBIter = MBB; 11453 ++MBBIter; 11454 11455 /// First build the CFG 11456 MachineFunction *F = MBB->getParent(); 11457 MachineBasicBlock *thisMBB = MBB; 11458 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 11459 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 11460 F->insert(MBBIter, newMBB); 11461 F->insert(MBBIter, nextMBB); 11462 11463 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 11464 nextMBB->splice(nextMBB->begin(), thisMBB, 11465 llvm::next(MachineBasicBlock::iterator(bInstr)), 11466 thisMBB->end()); 11467 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 11468 11469 // Update thisMBB to fall through to newMBB 11470 thisMBB->addSuccessor(newMBB); 11471 11472 // newMBB jumps to itself and fall through to nextMBB 11473 newMBB->addSuccessor(nextMBB); 11474 newMBB->addSuccessor(newMBB); 11475 11476 // Insert instructions into newMBB based on incoming instruction 11477 assert(bInstr->getNumOperands() < X86::AddrNumOperands + 4 && 11478 "unexpected number of operands"); 11479 DebugLoc dl = bInstr->getDebugLoc(); 11480 MachineOperand& destOper = bInstr->getOperand(0); 11481 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 11482 int numArgs = bInstr->getNumOperands() - 1; 11483 for (int i=0; i < numArgs; ++i) 11484 argOpers[i] = &bInstr->getOperand(i+1); 11485 11486 // x86 address has 4 operands: base, index, scale, and displacement 11487 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 11488 int valArgIndx = lastAddrIndx + 1; 11489 11490 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 11491 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1); 11492 for (int i=0; i <= lastAddrIndx; ++i) 11493 (*MIB).addOperand(*argOpers[i]); 11494 11495 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 11496 assert((argOpers[valArgIndx]->isReg() || 11497 argOpers[valArgIndx]->isImm()) && 11498 "invalid operand"); 11499 if (argOpers[valArgIndx]->isReg()) 11500 MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2); 11501 else 11502 MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2); 11503 MIB.addReg(t1); 11504 (*MIB).addOperand(*argOpers[valArgIndx]); 11505 11506 unsigned t3 = F->getRegInfo().createVirtualRegister(RC); 11507 if (Invert) { 11508 MIB = BuildMI(newMBB, dl, TII->get(notOpc), t3).addReg(t2); 11509 } 11510 else 11511 t3 = t2; 11512 11513 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), EAXreg); 11514 MIB.addReg(t1); 11515 11516 MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc)); 11517 for (int i=0; i <= lastAddrIndx; ++i) 11518 (*MIB).addOperand(*argOpers[i]); 11519 MIB.addReg(t3); 11520 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 11521 (*MIB).setMemRefs(bInstr->memoperands_begin(), 11522 bInstr->memoperands_end()); 11523 11524 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg()); 11525 MIB.addReg(EAXreg); 11526 11527 // insert branch 11528 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 11529 11530 bInstr->eraseFromParent(); // The pseudo instruction is gone now. 11531 return nextMBB; 11532} 11533 11534// private utility function: 64 bit atomics on 32 bit host. 11535MachineBasicBlock * 11536X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr, 11537 MachineBasicBlock *MBB, 11538 unsigned regOpcL, 11539 unsigned regOpcH, 11540 unsigned immOpcL, 11541 unsigned immOpcH, 11542 bool Invert) const { 11543 // For the atomic bitwise operator, we generate 11544 // thisMBB (instructions are in pairs, except cmpxchg8b) 11545 // ld t1,t2 = [bitinstr.addr] 11546 // newMBB: 11547 // out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4) 11548 // op t5, t6 <- out1, out2, [bitinstr.val] 11549 // (for SWAP, substitute: mov t5, t6 <- [bitinstr.val]) 11550 // neg t7, t8 < t5, t6 (if Invert) 11551 // mov ECX, EBX <- t5, t6 11552 // mov EAX, EDX <- t1, t2 11553 // cmpxchg8b [bitinstr.addr] [EAX, EDX, EBX, ECX implicit] 11554 // mov t3, t4 <- EAX, EDX 11555 // bz newMBB 11556 // result in out1, out2 11557 // fallthrough -->nextMBB 11558 11559 const TargetRegisterClass *RC = &X86::GR32RegClass; 11560 const unsigned LoadOpc = X86::MOV32rm; 11561 const unsigned NotOpc = X86::NOT32r; 11562 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11563 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 11564 MachineFunction::iterator MBBIter = MBB; 11565 ++MBBIter; 11566 11567 /// First build the CFG 11568 MachineFunction *F = MBB->getParent(); 11569 MachineBasicBlock *thisMBB = MBB; 11570 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 11571 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 11572 F->insert(MBBIter, newMBB); 11573 F->insert(MBBIter, nextMBB); 11574 11575 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 11576 nextMBB->splice(nextMBB->begin(), thisMBB, 11577 llvm::next(MachineBasicBlock::iterator(bInstr)), 11578 thisMBB->end()); 11579 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 11580 11581 // Update thisMBB to fall through to newMBB 11582 thisMBB->addSuccessor(newMBB); 11583 11584 // newMBB jumps to itself and fall through to nextMBB 11585 newMBB->addSuccessor(nextMBB); 11586 newMBB->addSuccessor(newMBB); 11587 11588 DebugLoc dl = bInstr->getDebugLoc(); 11589 // Insert instructions into newMBB based on incoming instruction 11590 // There are 8 "real" operands plus 9 implicit def/uses, ignored here. 11591 assert(bInstr->getNumOperands() < X86::AddrNumOperands + 14 && 11592 "unexpected number of operands"); 11593 MachineOperand& dest1Oper = bInstr->getOperand(0); 11594 MachineOperand& dest2Oper = bInstr->getOperand(1); 11595 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 11596 for (int i=0; i < 2 + X86::AddrNumOperands; ++i) { 11597 argOpers[i] = &bInstr->getOperand(i+2); 11598 11599 // We use some of the operands multiple times, so conservatively just 11600 // clear any kill flags that might be present. 11601 if (argOpers[i]->isReg() && argOpers[i]->isUse()) 11602 argOpers[i]->setIsKill(false); 11603 } 11604 11605 // x86 address has 5 operands: base, index, scale, displacement, and segment. 11606 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 11607 11608 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 11609 MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1); 11610 for (int i=0; i <= lastAddrIndx; ++i) 11611 (*MIB).addOperand(*argOpers[i]); 11612 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 11613 MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2); 11614 // add 4 to displacement. 11615 for (int i=0; i <= lastAddrIndx-2; ++i) 11616 (*MIB).addOperand(*argOpers[i]); 11617 MachineOperand newOp3 = *(argOpers[3]); 11618 if (newOp3.isImm()) 11619 newOp3.setImm(newOp3.getImm()+4); 11620 else 11621 newOp3.setOffset(newOp3.getOffset()+4); 11622 (*MIB).addOperand(newOp3); 11623 (*MIB).addOperand(*argOpers[lastAddrIndx]); 11624 11625 // t3/4 are defined later, at the bottom of the loop 11626 unsigned t3 = F->getRegInfo().createVirtualRegister(RC); 11627 unsigned t4 = F->getRegInfo().createVirtualRegister(RC); 11628 BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg()) 11629 .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB); 11630 BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg()) 11631 .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB); 11632 11633 // The subsequent operations should be using the destination registers of 11634 // the PHI instructions. 11635 t1 = dest1Oper.getReg(); 11636 t2 = dest2Oper.getReg(); 11637 11638 int valArgIndx = lastAddrIndx + 1; 11639 assert((argOpers[valArgIndx]->isReg() || 11640 argOpers[valArgIndx]->isImm()) && 11641 "invalid operand"); 11642 unsigned t5 = F->getRegInfo().createVirtualRegister(RC); 11643 unsigned t6 = F->getRegInfo().createVirtualRegister(RC); 11644 if (argOpers[valArgIndx]->isReg()) 11645 MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5); 11646 else 11647 MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5); 11648 if (regOpcL != X86::MOV32rr) 11649 MIB.addReg(t1); 11650 (*MIB).addOperand(*argOpers[valArgIndx]); 11651 assert(argOpers[valArgIndx + 1]->isReg() == 11652 argOpers[valArgIndx]->isReg()); 11653 assert(argOpers[valArgIndx + 1]->isImm() == 11654 argOpers[valArgIndx]->isImm()); 11655 if (argOpers[valArgIndx + 1]->isReg()) 11656 MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6); 11657 else 11658 MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6); 11659 if (regOpcH != X86::MOV32rr) 11660 MIB.addReg(t2); 11661 (*MIB).addOperand(*argOpers[valArgIndx + 1]); 11662 11663 unsigned t7, t8; 11664 if (Invert) { 11665 t7 = F->getRegInfo().createVirtualRegister(RC); 11666 t8 = F->getRegInfo().createVirtualRegister(RC); 11667 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t7).addReg(t5); 11668 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t8).addReg(t6); 11669 } else { 11670 t7 = t5; 11671 t8 = t6; 11672 } 11673 11674 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX); 11675 MIB.addReg(t1); 11676 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EDX); 11677 MIB.addReg(t2); 11678 11679 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EBX); 11680 MIB.addReg(t7); 11681 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::ECX); 11682 MIB.addReg(t8); 11683 11684 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B)); 11685 for (int i=0; i <= lastAddrIndx; ++i) 11686 (*MIB).addOperand(*argOpers[i]); 11687 11688 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 11689 (*MIB).setMemRefs(bInstr->memoperands_begin(), 11690 bInstr->memoperands_end()); 11691 11692 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t3); 11693 MIB.addReg(X86::EAX); 11694 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t4); 11695 MIB.addReg(X86::EDX); 11696 11697 // insert branch 11698 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 11699 11700 bInstr->eraseFromParent(); // The pseudo instruction is gone now. 11701 return nextMBB; 11702} 11703 11704// private utility function 11705MachineBasicBlock * 11706X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 11707 MachineBasicBlock *MBB, 11708 unsigned cmovOpc) const { 11709 // For the atomic min/max operator, we generate 11710 // thisMBB: 11711 // newMBB: 11712 // ld t1 = [min/max.addr] 11713 // mov t2 = [min/max.val] 11714 // cmp t1, t2 11715 // cmov[cond] t2 = t1 11716 // mov EAX = t1 11717 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 11718 // bz newMBB 11719 // fallthrough -->nextMBB 11720 // 11721 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11722 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 11723 MachineFunction::iterator MBBIter = MBB; 11724 ++MBBIter; 11725 11726 /// First build the CFG 11727 MachineFunction *F = MBB->getParent(); 11728 MachineBasicBlock *thisMBB = MBB; 11729 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 11730 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 11731 F->insert(MBBIter, newMBB); 11732 F->insert(MBBIter, nextMBB); 11733 11734 // Transfer the remainder of thisMBB and its successor edges to nextMBB. 11735 nextMBB->splice(nextMBB->begin(), thisMBB, 11736 llvm::next(MachineBasicBlock::iterator(mInstr)), 11737 thisMBB->end()); 11738 nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 11739 11740 // Update thisMBB to fall through to newMBB 11741 thisMBB->addSuccessor(newMBB); 11742 11743 // newMBB jumps to newMBB and fall through to nextMBB 11744 newMBB->addSuccessor(nextMBB); 11745 newMBB->addSuccessor(newMBB); 11746 11747 DebugLoc dl = mInstr->getDebugLoc(); 11748 // Insert instructions into newMBB based on incoming instruction 11749 assert(mInstr->getNumOperands() < X86::AddrNumOperands + 4 && 11750 "unexpected number of operands"); 11751 MachineOperand& destOper = mInstr->getOperand(0); 11752 MachineOperand* argOpers[2 + X86::AddrNumOperands]; 11753 int numArgs = mInstr->getNumOperands() - 1; 11754 for (int i=0; i < numArgs; ++i) 11755 argOpers[i] = &mInstr->getOperand(i+1); 11756 11757 // x86 address has 4 operands: base, index, scale, and displacement 11758 int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3] 11759 int valArgIndx = lastAddrIndx + 1; 11760 11761 unsigned t1 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass); 11762 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1); 11763 for (int i=0; i <= lastAddrIndx; ++i) 11764 (*MIB).addOperand(*argOpers[i]); 11765 11766 // We only support register and immediate values 11767 assert((argOpers[valArgIndx]->isReg() || 11768 argOpers[valArgIndx]->isImm()) && 11769 "invalid operand"); 11770 11771 unsigned t2 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass); 11772 if (argOpers[valArgIndx]->isReg()) 11773 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t2); 11774 else 11775 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2); 11776 (*MIB).addOperand(*argOpers[valArgIndx]); 11777 11778 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX); 11779 MIB.addReg(t1); 11780 11781 MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr)); 11782 MIB.addReg(t1); 11783 MIB.addReg(t2); 11784 11785 // Generate movc 11786 unsigned t3 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass); 11787 MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3); 11788 MIB.addReg(t2); 11789 MIB.addReg(t1); 11790 11791 // Cmp and exchange if none has modified the memory location 11792 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32)); 11793 for (int i=0; i <= lastAddrIndx; ++i) 11794 (*MIB).addOperand(*argOpers[i]); 11795 MIB.addReg(t3); 11796 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 11797 (*MIB).setMemRefs(mInstr->memoperands_begin(), 11798 mInstr->memoperands_end()); 11799 11800 MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg()); 11801 MIB.addReg(X86::EAX); 11802 11803 // insert branch 11804 BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB); 11805 11806 mInstr->eraseFromParent(); // The pseudo instruction is gone now. 11807 return nextMBB; 11808} 11809 11810// FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8 11811// or XMM0_V32I8 in AVX all of this code can be replaced with that 11812// in the .td file. 11813MachineBasicBlock * 11814X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB, 11815 unsigned numArgs, bool memArg) const { 11816 assert(Subtarget->hasSSE42() && 11817 "Target must have SSE4.2 or AVX features enabled"); 11818 11819 DebugLoc dl = MI->getDebugLoc(); 11820 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11821 unsigned Opc; 11822 if (!Subtarget->hasAVX()) { 11823 if (memArg) 11824 Opc = numArgs == 3 ? X86::PCMPISTRM128rm : X86::PCMPESTRM128rm; 11825 else 11826 Opc = numArgs == 3 ? X86::PCMPISTRM128rr : X86::PCMPESTRM128rr; 11827 } else { 11828 if (memArg) 11829 Opc = numArgs == 3 ? X86::VPCMPISTRM128rm : X86::VPCMPESTRM128rm; 11830 else 11831 Opc = numArgs == 3 ? X86::VPCMPISTRM128rr : X86::VPCMPESTRM128rr; 11832 } 11833 11834 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc)); 11835 for (unsigned i = 0; i < numArgs; ++i) { 11836 MachineOperand &Op = MI->getOperand(i+1); 11837 if (!(Op.isReg() && Op.isImplicit())) 11838 MIB.addOperand(Op); 11839 } 11840 BuildMI(*BB, MI, dl, 11841 TII->get(Subtarget->hasAVX() ? X86::VMOVAPSrr : X86::MOVAPSrr), 11842 MI->getOperand(0).getReg()) 11843 .addReg(X86::XMM0); 11844 11845 MI->eraseFromParent(); 11846 return BB; 11847} 11848 11849MachineBasicBlock * 11850X86TargetLowering::EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB) const { 11851 DebugLoc dl = MI->getDebugLoc(); 11852 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11853 11854 // Address into RAX/EAX, other two args into ECX, EDX. 11855 unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r; 11856 unsigned MemReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; 11857 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(MemOpc), MemReg); 11858 for (int i = 0; i < X86::AddrNumOperands; ++i) 11859 MIB.addOperand(MI->getOperand(i)); 11860 11861 unsigned ValOps = X86::AddrNumOperands; 11862 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX) 11863 .addReg(MI->getOperand(ValOps).getReg()); 11864 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EDX) 11865 .addReg(MI->getOperand(ValOps+1).getReg()); 11866 11867 // The instruction doesn't actually take any operands though. 11868 BuildMI(*BB, MI, dl, TII->get(X86::MONITORrrr)); 11869 11870 MI->eraseFromParent(); // The pseudo is gone now. 11871 return BB; 11872} 11873 11874MachineBasicBlock * 11875X86TargetLowering::EmitMwait(MachineInstr *MI, MachineBasicBlock *BB) const { 11876 DebugLoc dl = MI->getDebugLoc(); 11877 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11878 11879 // First arg in ECX, the second in EAX. 11880 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX) 11881 .addReg(MI->getOperand(0).getReg()); 11882 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EAX) 11883 .addReg(MI->getOperand(1).getReg()); 11884 11885 // The instruction doesn't actually take any operands though. 11886 BuildMI(*BB, MI, dl, TII->get(X86::MWAITrr)); 11887 11888 MI->eraseFromParent(); // The pseudo is gone now. 11889 return BB; 11890} 11891 11892MachineBasicBlock * 11893X86TargetLowering::EmitVAARG64WithCustomInserter( 11894 MachineInstr *MI, 11895 MachineBasicBlock *MBB) const { 11896 // Emit va_arg instruction on X86-64. 11897 11898 // Operands to this pseudo-instruction: 11899 // 0 ) Output : destination address (reg) 11900 // 1-5) Input : va_list address (addr, i64mem) 11901 // 6 ) ArgSize : Size (in bytes) of vararg type 11902 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset 11903 // 8 ) Align : Alignment of type 11904 // 9 ) EFLAGS (implicit-def) 11905 11906 assert(MI->getNumOperands() == 10 && "VAARG_64 should have 10 operands!"); 11907 assert(X86::AddrNumOperands == 5 && "VAARG_64 assumes 5 address operands"); 11908 11909 unsigned DestReg = MI->getOperand(0).getReg(); 11910 MachineOperand &Base = MI->getOperand(1); 11911 MachineOperand &Scale = MI->getOperand(2); 11912 MachineOperand &Index = MI->getOperand(3); 11913 MachineOperand &Disp = MI->getOperand(4); 11914 MachineOperand &Segment = MI->getOperand(5); 11915 unsigned ArgSize = MI->getOperand(6).getImm(); 11916 unsigned ArgMode = MI->getOperand(7).getImm(); 11917 unsigned Align = MI->getOperand(8).getImm(); 11918 11919 // Memory Reference 11920 assert(MI->hasOneMemOperand() && "Expected VAARG_64 to have one memoperand"); 11921 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 11922 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 11923 11924 // Machine Information 11925 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 11926 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 11927 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); 11928 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); 11929 DebugLoc DL = MI->getDebugLoc(); 11930 11931 // struct va_list { 11932 // i32 gp_offset 11933 // i32 fp_offset 11934 // i64 overflow_area (address) 11935 // i64 reg_save_area (address) 11936 // } 11937 // sizeof(va_list) = 24 11938 // alignment(va_list) = 8 11939 11940 unsigned TotalNumIntRegs = 6; 11941 unsigned TotalNumXMMRegs = 8; 11942 bool UseGPOffset = (ArgMode == 1); 11943 bool UseFPOffset = (ArgMode == 2); 11944 unsigned MaxOffset = TotalNumIntRegs * 8 + 11945 (UseFPOffset ? TotalNumXMMRegs * 16 : 0); 11946 11947 /* Align ArgSize to a multiple of 8 */ 11948 unsigned ArgSizeA8 = (ArgSize + 7) & ~7; 11949 bool NeedsAlign = (Align > 8); 11950 11951 MachineBasicBlock *thisMBB = MBB; 11952 MachineBasicBlock *overflowMBB; 11953 MachineBasicBlock *offsetMBB; 11954 MachineBasicBlock *endMBB; 11955 11956 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB 11957 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB 11958 unsigned OffsetReg = 0; 11959 11960 if (!UseGPOffset && !UseFPOffset) { 11961 // If we only pull from the overflow region, we don't create a branch. 11962 // We don't need to alter control flow. 11963 OffsetDestReg = 0; // unused 11964 OverflowDestReg = DestReg; 11965 11966 offsetMBB = NULL; 11967 overflowMBB = thisMBB; 11968 endMBB = thisMBB; 11969 } else { 11970 // First emit code to check if gp_offset (or fp_offset) is below the bound. 11971 // If so, pull the argument from reg_save_area. (branch to offsetMBB) 11972 // If not, pull from overflow_area. (branch to overflowMBB) 11973 // 11974 // thisMBB 11975 // | . 11976 // | . 11977 // offsetMBB overflowMBB 11978 // | . 11979 // | . 11980 // endMBB 11981 11982 // Registers for the PHI in endMBB 11983 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass); 11984 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass); 11985 11986 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 11987 MachineFunction *MF = MBB->getParent(); 11988 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB); 11989 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB); 11990 endMBB = MF->CreateMachineBasicBlock(LLVM_BB); 11991 11992 MachineFunction::iterator MBBIter = MBB; 11993 ++MBBIter; 11994 11995 // Insert the new basic blocks 11996 MF->insert(MBBIter, offsetMBB); 11997 MF->insert(MBBIter, overflowMBB); 11998 MF->insert(MBBIter, endMBB); 11999 12000 // Transfer the remainder of MBB and its successor edges to endMBB. 12001 endMBB->splice(endMBB->begin(), thisMBB, 12002 llvm::next(MachineBasicBlock::iterator(MI)), 12003 thisMBB->end()); 12004 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB); 12005 12006 // Make offsetMBB and overflowMBB successors of thisMBB 12007 thisMBB->addSuccessor(offsetMBB); 12008 thisMBB->addSuccessor(overflowMBB); 12009 12010 // endMBB is a successor of both offsetMBB and overflowMBB 12011 offsetMBB->addSuccessor(endMBB); 12012 overflowMBB->addSuccessor(endMBB); 12013 12014 // Load the offset value into a register 12015 OffsetReg = MRI.createVirtualRegister(OffsetRegClass); 12016 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg) 12017 .addOperand(Base) 12018 .addOperand(Scale) 12019 .addOperand(Index) 12020 .addDisp(Disp, UseFPOffset ? 4 : 0) 12021 .addOperand(Segment) 12022 .setMemRefs(MMOBegin, MMOEnd); 12023 12024 // Check if there is enough room left to pull this argument. 12025 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri)) 12026 .addReg(OffsetReg) 12027 .addImm(MaxOffset + 8 - ArgSizeA8); 12028 12029 // Branch to "overflowMBB" if offset >= max 12030 // Fall through to "offsetMBB" otherwise 12031 BuildMI(thisMBB, DL, TII->get(X86::GetCondBranchFromCond(X86::COND_AE))) 12032 .addMBB(overflowMBB); 12033 } 12034 12035 // In offsetMBB, emit code to use the reg_save_area. 12036 if (offsetMBB) { 12037 assert(OffsetReg != 0); 12038 12039 // Read the reg_save_area address. 12040 unsigned RegSaveReg = MRI.createVirtualRegister(AddrRegClass); 12041 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg) 12042 .addOperand(Base) 12043 .addOperand(Scale) 12044 .addOperand(Index) 12045 .addDisp(Disp, 16) 12046 .addOperand(Segment) 12047 .setMemRefs(MMOBegin, MMOEnd); 12048 12049 // Zero-extend the offset 12050 unsigned OffsetReg64 = MRI.createVirtualRegister(AddrRegClass); 12051 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64) 12052 .addImm(0) 12053 .addReg(OffsetReg) 12054 .addImm(X86::sub_32bit); 12055 12056 // Add the offset to the reg_save_area to get the final address. 12057 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg) 12058 .addReg(OffsetReg64) 12059 .addReg(RegSaveReg); 12060 12061 // Compute the offset for the next argument 12062 unsigned NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass); 12063 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg) 12064 .addReg(OffsetReg) 12065 .addImm(UseFPOffset ? 16 : 8); 12066 12067 // Store it back into the va_list. 12068 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr)) 12069 .addOperand(Base) 12070 .addOperand(Scale) 12071 .addOperand(Index) 12072 .addDisp(Disp, UseFPOffset ? 4 : 0) 12073 .addOperand(Segment) 12074 .addReg(NextOffsetReg) 12075 .setMemRefs(MMOBegin, MMOEnd); 12076 12077 // Jump to endMBB 12078 BuildMI(offsetMBB, DL, TII->get(X86::JMP_4)) 12079 .addMBB(endMBB); 12080 } 12081 12082 // 12083 // Emit code to use overflow area 12084 // 12085 12086 // Load the overflow_area address into a register. 12087 unsigned OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass); 12088 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg) 12089 .addOperand(Base) 12090 .addOperand(Scale) 12091 .addOperand(Index) 12092 .addDisp(Disp, 8) 12093 .addOperand(Segment) 12094 .setMemRefs(MMOBegin, MMOEnd); 12095 12096 // If we need to align it, do so. Otherwise, just copy the address 12097 // to OverflowDestReg. 12098 if (NeedsAlign) { 12099 // Align the overflow address 12100 assert((Align & (Align-1)) == 0 && "Alignment must be a power of 2"); 12101 unsigned TmpReg = MRI.createVirtualRegister(AddrRegClass); 12102 12103 // aligned_addr = (addr + (align-1)) & ~(align-1) 12104 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg) 12105 .addReg(OverflowAddrReg) 12106 .addImm(Align-1); 12107 12108 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg) 12109 .addReg(TmpReg) 12110 .addImm(~(uint64_t)(Align-1)); 12111 } else { 12112 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg) 12113 .addReg(OverflowAddrReg); 12114 } 12115 12116 // Compute the next overflow address after this argument. 12117 // (the overflow address should be kept 8-byte aligned) 12118 unsigned NextAddrReg = MRI.createVirtualRegister(AddrRegClass); 12119 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg) 12120 .addReg(OverflowDestReg) 12121 .addImm(ArgSizeA8); 12122 12123 // Store the new overflow address. 12124 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr)) 12125 .addOperand(Base) 12126 .addOperand(Scale) 12127 .addOperand(Index) 12128 .addDisp(Disp, 8) 12129 .addOperand(Segment) 12130 .addReg(NextAddrReg) 12131 .setMemRefs(MMOBegin, MMOEnd); 12132 12133 // If we branched, emit the PHI to the front of endMBB. 12134 if (offsetMBB) { 12135 BuildMI(*endMBB, endMBB->begin(), DL, 12136 TII->get(X86::PHI), DestReg) 12137 .addReg(OffsetDestReg).addMBB(offsetMBB) 12138 .addReg(OverflowDestReg).addMBB(overflowMBB); 12139 } 12140 12141 // Erase the pseudo instruction 12142 MI->eraseFromParent(); 12143 12144 return endMBB; 12145} 12146 12147MachineBasicBlock * 12148X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( 12149 MachineInstr *MI, 12150 MachineBasicBlock *MBB) const { 12151 // Emit code to save XMM registers to the stack. The ABI says that the 12152 // number of registers to save is given in %al, so it's theoretically 12153 // possible to do an indirect jump trick to avoid saving all of them, 12154 // however this code takes a simpler approach and just executes all 12155 // of the stores if %al is non-zero. It's less code, and it's probably 12156 // easier on the hardware branch predictor, and stores aren't all that 12157 // expensive anyway. 12158 12159 // Create the new basic blocks. One block contains all the XMM stores, 12160 // and one block is the final destination regardless of whether any 12161 // stores were performed. 12162 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 12163 MachineFunction *F = MBB->getParent(); 12164 MachineFunction::iterator MBBIter = MBB; 12165 ++MBBIter; 12166 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB); 12167 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB); 12168 F->insert(MBBIter, XMMSaveMBB); 12169 F->insert(MBBIter, EndMBB); 12170 12171 // Transfer the remainder of MBB and its successor edges to EndMBB. 12172 EndMBB->splice(EndMBB->begin(), MBB, 12173 llvm::next(MachineBasicBlock::iterator(MI)), 12174 MBB->end()); 12175 EndMBB->transferSuccessorsAndUpdatePHIs(MBB); 12176 12177 // The original block will now fall through to the XMM save block. 12178 MBB->addSuccessor(XMMSaveMBB); 12179 // The XMMSaveMBB will fall through to the end block. 12180 XMMSaveMBB->addSuccessor(EndMBB); 12181 12182 // Now add the instructions. 12183 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 12184 DebugLoc DL = MI->getDebugLoc(); 12185 12186 unsigned CountReg = MI->getOperand(0).getReg(); 12187 int64_t RegSaveFrameIndex = MI->getOperand(1).getImm(); 12188 int64_t VarArgsFPOffset = MI->getOperand(2).getImm(); 12189 12190 if (!Subtarget->isTargetWin64()) { 12191 // If %al is 0, branch around the XMM save block. 12192 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg); 12193 BuildMI(MBB, DL, TII->get(X86::JE_4)).addMBB(EndMBB); 12194 MBB->addSuccessor(EndMBB); 12195 } 12196 12197 unsigned MOVOpc = Subtarget->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr; 12198 // In the XMM save block, save all the XMM argument registers. 12199 for (int i = 3, e = MI->getNumOperands(); i != e; ++i) { 12200 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset; 12201 MachineMemOperand *MMO = 12202 F->getMachineMemOperand( 12203 MachinePointerInfo::getFixedStack(RegSaveFrameIndex, Offset), 12204 MachineMemOperand::MOStore, 12205 /*Size=*/16, /*Align=*/16); 12206 BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc)) 12207 .addFrameIndex(RegSaveFrameIndex) 12208 .addImm(/*Scale=*/1) 12209 .addReg(/*IndexReg=*/0) 12210 .addImm(/*Disp=*/Offset) 12211 .addReg(/*Segment=*/0) 12212 .addReg(MI->getOperand(i).getReg()) 12213 .addMemOperand(MMO); 12214 } 12215 12216 MI->eraseFromParent(); // The pseudo instruction is gone now. 12217 12218 return EndMBB; 12219} 12220 12221// The EFLAGS operand of SelectItr might be missing a kill marker 12222// because there were multiple uses of EFLAGS, and ISel didn't know 12223// which to mark. Figure out whether SelectItr should have had a 12224// kill marker, and set it if it should. Returns the correct kill 12225// marker value. 12226static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr, 12227 MachineBasicBlock* BB, 12228 const TargetRegisterInfo* TRI) { 12229 // Scan forward through BB for a use/def of EFLAGS. 12230 MachineBasicBlock::iterator miI(llvm::next(SelectItr)); 12231 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) { 12232 const MachineInstr& mi = *miI; 12233 if (mi.readsRegister(X86::EFLAGS)) 12234 return false; 12235 if (mi.definesRegister(X86::EFLAGS)) 12236 break; // Should have kill-flag - update below. 12237 } 12238 12239 // If we hit the end of the block, check whether EFLAGS is live into a 12240 // successor. 12241 if (miI == BB->end()) { 12242 for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(), 12243 sEnd = BB->succ_end(); 12244 sItr != sEnd; ++sItr) { 12245 MachineBasicBlock* succ = *sItr; 12246 if (succ->isLiveIn(X86::EFLAGS)) 12247 return false; 12248 } 12249 } 12250 12251 // We found a def, or hit the end of the basic block and EFLAGS wasn't live 12252 // out. SelectMI should have a kill flag on EFLAGS. 12253 SelectItr->addRegisterKilled(X86::EFLAGS, TRI); 12254 return true; 12255} 12256 12257MachineBasicBlock * 12258X86TargetLowering::EmitLoweredSelect(MachineInstr *MI, 12259 MachineBasicBlock *BB) const { 12260 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 12261 DebugLoc DL = MI->getDebugLoc(); 12262 12263 // To "insert" a SELECT_CC instruction, we actually have to insert the 12264 // diamond control-flow pattern. The incoming instruction knows the 12265 // destination vreg to set, the condition code register to branch on, the 12266 // true/false values to select between, and a branch opcode to use. 12267 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 12268 MachineFunction::iterator It = BB; 12269 ++It; 12270 12271 // thisMBB: 12272 // ... 12273 // TrueVal = ... 12274 // cmpTY ccX, r1, r2 12275 // bCC copy1MBB 12276 // fallthrough --> copy0MBB 12277 MachineBasicBlock *thisMBB = BB; 12278 MachineFunction *F = BB->getParent(); 12279 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 12280 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 12281 F->insert(It, copy0MBB); 12282 F->insert(It, sinkMBB); 12283 12284 // If the EFLAGS register isn't dead in the terminator, then claim that it's 12285 // live into the sink and copy blocks. 12286 const TargetRegisterInfo* TRI = getTargetMachine().getRegisterInfo(); 12287 if (!MI->killsRegister(X86::EFLAGS) && 12288 !checkAndUpdateEFLAGSKill(MI, BB, TRI)) { 12289 copy0MBB->addLiveIn(X86::EFLAGS); 12290 sinkMBB->addLiveIn(X86::EFLAGS); 12291 } 12292 12293 // Transfer the remainder of BB and its successor edges to sinkMBB. 12294 sinkMBB->splice(sinkMBB->begin(), BB, 12295 llvm::next(MachineBasicBlock::iterator(MI)), 12296 BB->end()); 12297 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 12298 12299 // Add the true and fallthrough blocks as its successors. 12300 BB->addSuccessor(copy0MBB); 12301 BB->addSuccessor(sinkMBB); 12302 12303 // Create the conditional branch instruction. 12304 unsigned Opc = 12305 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 12306 BuildMI(BB, DL, TII->get(Opc)).addMBB(sinkMBB); 12307 12308 // copy0MBB: 12309 // %FalseValue = ... 12310 // # fallthrough to sinkMBB 12311 copy0MBB->addSuccessor(sinkMBB); 12312 12313 // sinkMBB: 12314 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 12315 // ... 12316 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 12317 TII->get(X86::PHI), MI->getOperand(0).getReg()) 12318 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 12319 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 12320 12321 MI->eraseFromParent(); // The pseudo instruction is gone now. 12322 return sinkMBB; 12323} 12324 12325MachineBasicBlock * 12326X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI, MachineBasicBlock *BB, 12327 bool Is64Bit) const { 12328 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 12329 DebugLoc DL = MI->getDebugLoc(); 12330 MachineFunction *MF = BB->getParent(); 12331 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 12332 12333 assert(getTargetMachine().Options.EnableSegmentedStacks); 12334 12335 unsigned TlsReg = Is64Bit ? X86::FS : X86::GS; 12336 unsigned TlsOffset = Is64Bit ? 0x70 : 0x30; 12337 12338 // BB: 12339 // ... [Till the alloca] 12340 // If stacklet is not large enough, jump to mallocMBB 12341 // 12342 // bumpMBB: 12343 // Allocate by subtracting from RSP 12344 // Jump to continueMBB 12345 // 12346 // mallocMBB: 12347 // Allocate by call to runtime 12348 // 12349 // continueMBB: 12350 // ... 12351 // [rest of original BB] 12352 // 12353 12354 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB); 12355 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB); 12356 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB); 12357 12358 MachineRegisterInfo &MRI = MF->getRegInfo(); 12359 const TargetRegisterClass *AddrRegClass = 12360 getRegClassFor(Is64Bit ? MVT::i64:MVT::i32); 12361 12362 unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass), 12363 bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass), 12364 tmpSPVReg = MRI.createVirtualRegister(AddrRegClass), 12365 SPLimitVReg = MRI.createVirtualRegister(AddrRegClass), 12366 sizeVReg = MI->getOperand(1).getReg(), 12367 physSPReg = Is64Bit ? X86::RSP : X86::ESP; 12368 12369 MachineFunction::iterator MBBIter = BB; 12370 ++MBBIter; 12371 12372 MF->insert(MBBIter, bumpMBB); 12373 MF->insert(MBBIter, mallocMBB); 12374 MF->insert(MBBIter, continueMBB); 12375 12376 continueMBB->splice(continueMBB->begin(), BB, llvm::next 12377 (MachineBasicBlock::iterator(MI)), BB->end()); 12378 continueMBB->transferSuccessorsAndUpdatePHIs(BB); 12379 12380 // Add code to the main basic block to check if the stack limit has been hit, 12381 // and if so, jump to mallocMBB otherwise to bumpMBB. 12382 BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg); 12383 BuildMI(BB, DL, TII->get(Is64Bit ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg) 12384 .addReg(tmpSPVReg).addReg(sizeVReg); 12385 BuildMI(BB, DL, TII->get(Is64Bit ? X86::CMP64mr:X86::CMP32mr)) 12386 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg) 12387 .addReg(SPLimitVReg); 12388 BuildMI(BB, DL, TII->get(X86::JG_4)).addMBB(mallocMBB); 12389 12390 // bumpMBB simply decreases the stack pointer, since we know the current 12391 // stacklet has enough space. 12392 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg) 12393 .addReg(SPLimitVReg); 12394 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg) 12395 .addReg(SPLimitVReg); 12396 BuildMI(bumpMBB, DL, TII->get(X86::JMP_4)).addMBB(continueMBB); 12397 12398 // Calls into a routine in libgcc to allocate more space from the heap. 12399 const uint32_t *RegMask = 12400 getTargetMachine().getRegisterInfo()->getCallPreservedMask(CallingConv::C); 12401 if (Is64Bit) { 12402 BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI) 12403 .addReg(sizeVReg); 12404 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32)) 12405 .addExternalSymbol("__morestack_allocate_stack_space").addReg(X86::RDI) 12406 .addRegMask(RegMask) 12407 .addReg(X86::RAX, RegState::ImplicitDefine); 12408 } else { 12409 BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg) 12410 .addImm(12); 12411 BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg); 12412 BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32)) 12413 .addExternalSymbol("__morestack_allocate_stack_space") 12414 .addRegMask(RegMask) 12415 .addReg(X86::EAX, RegState::ImplicitDefine); 12416 } 12417 12418 if (!Is64Bit) 12419 BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg) 12420 .addImm(16); 12421 12422 BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg) 12423 .addReg(Is64Bit ? X86::RAX : X86::EAX); 12424 BuildMI(mallocMBB, DL, TII->get(X86::JMP_4)).addMBB(continueMBB); 12425 12426 // Set up the CFG correctly. 12427 BB->addSuccessor(bumpMBB); 12428 BB->addSuccessor(mallocMBB); 12429 mallocMBB->addSuccessor(continueMBB); 12430 bumpMBB->addSuccessor(continueMBB); 12431 12432 // Take care of the PHI nodes. 12433 BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI), 12434 MI->getOperand(0).getReg()) 12435 .addReg(mallocPtrVReg).addMBB(mallocMBB) 12436 .addReg(bumpSPPtrVReg).addMBB(bumpMBB); 12437 12438 // Delete the original pseudo instruction. 12439 MI->eraseFromParent(); 12440 12441 // And we're done. 12442 return continueMBB; 12443} 12444 12445MachineBasicBlock * 12446X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI, 12447 MachineBasicBlock *BB) const { 12448 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 12449 DebugLoc DL = MI->getDebugLoc(); 12450 12451 assert(!Subtarget->isTargetEnvMacho()); 12452 12453 // The lowering is pretty easy: we're just emitting the call to _alloca. The 12454 // non-trivial part is impdef of ESP. 12455 12456 if (Subtarget->isTargetWin64()) { 12457 if (Subtarget->isTargetCygMing()) { 12458 // ___chkstk(Mingw64): 12459 // Clobbers R10, R11, RAX and EFLAGS. 12460 // Updates RSP. 12461 BuildMI(*BB, MI, DL, TII->get(X86::W64ALLOCA)) 12462 .addExternalSymbol("___chkstk") 12463 .addReg(X86::RAX, RegState::Implicit) 12464 .addReg(X86::RSP, RegState::Implicit) 12465 .addReg(X86::RAX, RegState::Define | RegState::Implicit) 12466 .addReg(X86::RSP, RegState::Define | RegState::Implicit) 12467 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 12468 } else { 12469 // __chkstk(MSVCRT): does not update stack pointer. 12470 // Clobbers R10, R11 and EFLAGS. 12471 // FIXME: RAX(allocated size) might be reused and not killed. 12472 BuildMI(*BB, MI, DL, TII->get(X86::W64ALLOCA)) 12473 .addExternalSymbol("__chkstk") 12474 .addReg(X86::RAX, RegState::Implicit) 12475 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 12476 // RAX has the offset to subtracted from RSP. 12477 BuildMI(*BB, MI, DL, TII->get(X86::SUB64rr), X86::RSP) 12478 .addReg(X86::RSP) 12479 .addReg(X86::RAX); 12480 } 12481 } else { 12482 const char *StackProbeSymbol = 12483 Subtarget->isTargetWindows() ? "_chkstk" : "_alloca"; 12484 12485 BuildMI(*BB, MI, DL, TII->get(X86::CALLpcrel32)) 12486 .addExternalSymbol(StackProbeSymbol) 12487 .addReg(X86::EAX, RegState::Implicit) 12488 .addReg(X86::ESP, RegState::Implicit) 12489 .addReg(X86::EAX, RegState::Define | RegState::Implicit) 12490 .addReg(X86::ESP, RegState::Define | RegState::Implicit) 12491 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 12492 } 12493 12494 MI->eraseFromParent(); // The pseudo instruction is gone now. 12495 return BB; 12496} 12497 12498MachineBasicBlock * 12499X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI, 12500 MachineBasicBlock *BB) const { 12501 // This is pretty easy. We're taking the value that we received from 12502 // our load from the relocation, sticking it in either RDI (x86-64) 12503 // or EAX and doing an indirect call. The return value will then 12504 // be in the normal return register. 12505 const X86InstrInfo *TII 12506 = static_cast<const X86InstrInfo*>(getTargetMachine().getInstrInfo()); 12507 DebugLoc DL = MI->getDebugLoc(); 12508 MachineFunction *F = BB->getParent(); 12509 12510 assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?"); 12511 assert(MI->getOperand(3).isGlobal() && "This should be a global"); 12512 12513 // Get a register mask for the lowered call. 12514 // FIXME: The 32-bit calls have non-standard calling conventions. Use a 12515 // proper register mask. 12516 const uint32_t *RegMask = 12517 getTargetMachine().getRegisterInfo()->getCallPreservedMask(CallingConv::C); 12518 if (Subtarget->is64Bit()) { 12519 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 12520 TII->get(X86::MOV64rm), X86::RDI) 12521 .addReg(X86::RIP) 12522 .addImm(0).addReg(0) 12523 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 12524 MI->getOperand(3).getTargetFlags()) 12525 .addReg(0); 12526 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m)); 12527 addDirectMem(MIB, X86::RDI); 12528 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask); 12529 } else if (getTargetMachine().getRelocationModel() != Reloc::PIC_) { 12530 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 12531 TII->get(X86::MOV32rm), X86::EAX) 12532 .addReg(0) 12533 .addImm(0).addReg(0) 12534 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 12535 MI->getOperand(3).getTargetFlags()) 12536 .addReg(0); 12537 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m)); 12538 addDirectMem(MIB, X86::EAX); 12539 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask); 12540 } else { 12541 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, 12542 TII->get(X86::MOV32rm), X86::EAX) 12543 .addReg(TII->getGlobalBaseReg(F)) 12544 .addImm(0).addReg(0) 12545 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0, 12546 MI->getOperand(3).getTargetFlags()) 12547 .addReg(0); 12548 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m)); 12549 addDirectMem(MIB, X86::EAX); 12550 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask); 12551 } 12552 12553 MI->eraseFromParent(); // The pseudo instruction is gone now. 12554 return BB; 12555} 12556 12557MachineBasicBlock * 12558X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 12559 MachineBasicBlock *BB) const { 12560 switch (MI->getOpcode()) { 12561 default: llvm_unreachable("Unexpected instr type to insert"); 12562 case X86::TAILJMPd64: 12563 case X86::TAILJMPr64: 12564 case X86::TAILJMPm64: 12565 llvm_unreachable("TAILJMP64 would not be touched here."); 12566 case X86::TCRETURNdi64: 12567 case X86::TCRETURNri64: 12568 case X86::TCRETURNmi64: 12569 return BB; 12570 case X86::WIN_ALLOCA: 12571 return EmitLoweredWinAlloca(MI, BB); 12572 case X86::SEG_ALLOCA_32: 12573 return EmitLoweredSegAlloca(MI, BB, false); 12574 case X86::SEG_ALLOCA_64: 12575 return EmitLoweredSegAlloca(MI, BB, true); 12576 case X86::TLSCall_32: 12577 case X86::TLSCall_64: 12578 return EmitLoweredTLSCall(MI, BB); 12579 case X86::CMOV_GR8: 12580 case X86::CMOV_FR32: 12581 case X86::CMOV_FR64: 12582 case X86::CMOV_V4F32: 12583 case X86::CMOV_V2F64: 12584 case X86::CMOV_V2I64: 12585 case X86::CMOV_V8F32: 12586 case X86::CMOV_V4F64: 12587 case X86::CMOV_V4I64: 12588 case X86::CMOV_GR16: 12589 case X86::CMOV_GR32: 12590 case X86::CMOV_RFP32: 12591 case X86::CMOV_RFP64: 12592 case X86::CMOV_RFP80: 12593 return EmitLoweredSelect(MI, BB); 12594 12595 case X86::FP32_TO_INT16_IN_MEM: 12596 case X86::FP32_TO_INT32_IN_MEM: 12597 case X86::FP32_TO_INT64_IN_MEM: 12598 case X86::FP64_TO_INT16_IN_MEM: 12599 case X86::FP64_TO_INT32_IN_MEM: 12600 case X86::FP64_TO_INT64_IN_MEM: 12601 case X86::FP80_TO_INT16_IN_MEM: 12602 case X86::FP80_TO_INT32_IN_MEM: 12603 case X86::FP80_TO_INT64_IN_MEM: { 12604 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 12605 DebugLoc DL = MI->getDebugLoc(); 12606 12607 // Change the floating point control register to use "round towards zero" 12608 // mode when truncating to an integer value. 12609 MachineFunction *F = BB->getParent(); 12610 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2, false); 12611 addFrameReference(BuildMI(*BB, MI, DL, 12612 TII->get(X86::FNSTCW16m)), CWFrameIdx); 12613 12614 // Load the old value of the high byte of the control word... 12615 unsigned OldCW = 12616 F->getRegInfo().createVirtualRegister(&X86::GR16RegClass); 12617 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16rm), OldCW), 12618 CWFrameIdx); 12619 12620 // Set the high part to be round to zero... 12621 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mi)), CWFrameIdx) 12622 .addImm(0xC7F); 12623 12624 // Reload the modified control word now... 12625 addFrameReference(BuildMI(*BB, MI, DL, 12626 TII->get(X86::FLDCW16m)), CWFrameIdx); 12627 12628 // Restore the memory image of control word to original value 12629 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)), CWFrameIdx) 12630 .addReg(OldCW); 12631 12632 // Get the X86 opcode to use. 12633 unsigned Opc; 12634 switch (MI->getOpcode()) { 12635 default: llvm_unreachable("illegal opcode!"); 12636 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 12637 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 12638 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 12639 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 12640 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 12641 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 12642 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 12643 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 12644 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 12645 } 12646 12647 X86AddressMode AM; 12648 MachineOperand &Op = MI->getOperand(0); 12649 if (Op.isReg()) { 12650 AM.BaseType = X86AddressMode::RegBase; 12651 AM.Base.Reg = Op.getReg(); 12652 } else { 12653 AM.BaseType = X86AddressMode::FrameIndexBase; 12654 AM.Base.FrameIndex = Op.getIndex(); 12655 } 12656 Op = MI->getOperand(1); 12657 if (Op.isImm()) 12658 AM.Scale = Op.getImm(); 12659 Op = MI->getOperand(2); 12660 if (Op.isImm()) 12661 AM.IndexReg = Op.getImm(); 12662 Op = MI->getOperand(3); 12663 if (Op.isGlobal()) { 12664 AM.GV = Op.getGlobal(); 12665 } else { 12666 AM.Disp = Op.getImm(); 12667 } 12668 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM) 12669 .addReg(MI->getOperand(X86::AddrNumOperands).getReg()); 12670 12671 // Reload the original control word now. 12672 addFrameReference(BuildMI(*BB, MI, DL, 12673 TII->get(X86::FLDCW16m)), CWFrameIdx); 12674 12675 MI->eraseFromParent(); // The pseudo instruction is gone now. 12676 return BB; 12677 } 12678 // String/text processing lowering. 12679 case X86::PCMPISTRM128REG: 12680 case X86::VPCMPISTRM128REG: 12681 return EmitPCMP(MI, BB, 3, false /* in-mem */); 12682 case X86::PCMPISTRM128MEM: 12683 case X86::VPCMPISTRM128MEM: 12684 return EmitPCMP(MI, BB, 3, true /* in-mem */); 12685 case X86::PCMPESTRM128REG: 12686 case X86::VPCMPESTRM128REG: 12687 return EmitPCMP(MI, BB, 5, false /* in mem */); 12688 case X86::PCMPESTRM128MEM: 12689 case X86::VPCMPESTRM128MEM: 12690 return EmitPCMP(MI, BB, 5, true /* in mem */); 12691 12692 // Thread synchronization. 12693 case X86::MONITOR: 12694 return EmitMonitor(MI, BB); 12695 case X86::MWAIT: 12696 return EmitMwait(MI, BB); 12697 12698 // Atomic Lowering. 12699 case X86::ATOMAND32: 12700 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 12701 X86::AND32ri, X86::MOV32rm, 12702 X86::LCMPXCHG32, 12703 X86::NOT32r, X86::EAX, 12704 &X86::GR32RegClass); 12705 case X86::ATOMOR32: 12706 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 12707 X86::OR32ri, X86::MOV32rm, 12708 X86::LCMPXCHG32, 12709 X86::NOT32r, X86::EAX, 12710 &X86::GR32RegClass); 12711 case X86::ATOMXOR32: 12712 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 12713 X86::XOR32ri, X86::MOV32rm, 12714 X86::LCMPXCHG32, 12715 X86::NOT32r, X86::EAX, 12716 &X86::GR32RegClass); 12717 case X86::ATOMNAND32: 12718 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 12719 X86::AND32ri, X86::MOV32rm, 12720 X86::LCMPXCHG32, 12721 X86::NOT32r, X86::EAX, 12722 &X86::GR32RegClass, true); 12723 case X86::ATOMMIN32: 12724 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 12725 case X86::ATOMMAX32: 12726 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 12727 case X86::ATOMUMIN32: 12728 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 12729 case X86::ATOMUMAX32: 12730 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 12731 12732 case X86::ATOMAND16: 12733 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 12734 X86::AND16ri, X86::MOV16rm, 12735 X86::LCMPXCHG16, 12736 X86::NOT16r, X86::AX, 12737 &X86::GR16RegClass); 12738 case X86::ATOMOR16: 12739 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 12740 X86::OR16ri, X86::MOV16rm, 12741 X86::LCMPXCHG16, 12742 X86::NOT16r, X86::AX, 12743 &X86::GR16RegClass); 12744 case X86::ATOMXOR16: 12745 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 12746 X86::XOR16ri, X86::MOV16rm, 12747 X86::LCMPXCHG16, 12748 X86::NOT16r, X86::AX, 12749 &X86::GR16RegClass); 12750 case X86::ATOMNAND16: 12751 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 12752 X86::AND16ri, X86::MOV16rm, 12753 X86::LCMPXCHG16, 12754 X86::NOT16r, X86::AX, 12755 &X86::GR16RegClass, true); 12756 case X86::ATOMMIN16: 12757 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 12758 case X86::ATOMMAX16: 12759 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 12760 case X86::ATOMUMIN16: 12761 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 12762 case X86::ATOMUMAX16: 12763 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 12764 12765 case X86::ATOMAND8: 12766 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 12767 X86::AND8ri, X86::MOV8rm, 12768 X86::LCMPXCHG8, 12769 X86::NOT8r, X86::AL, 12770 &X86::GR8RegClass); 12771 case X86::ATOMOR8: 12772 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 12773 X86::OR8ri, X86::MOV8rm, 12774 X86::LCMPXCHG8, 12775 X86::NOT8r, X86::AL, 12776 &X86::GR8RegClass); 12777 case X86::ATOMXOR8: 12778 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 12779 X86::XOR8ri, X86::MOV8rm, 12780 X86::LCMPXCHG8, 12781 X86::NOT8r, X86::AL, 12782 &X86::GR8RegClass); 12783 case X86::ATOMNAND8: 12784 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 12785 X86::AND8ri, X86::MOV8rm, 12786 X86::LCMPXCHG8, 12787 X86::NOT8r, X86::AL, 12788 &X86::GR8RegClass, true); 12789 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 12790 // This group is for 64-bit host. 12791 case X86::ATOMAND64: 12792 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 12793 X86::AND64ri32, X86::MOV64rm, 12794 X86::LCMPXCHG64, 12795 X86::NOT64r, X86::RAX, 12796 &X86::GR64RegClass); 12797 case X86::ATOMOR64: 12798 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, 12799 X86::OR64ri32, X86::MOV64rm, 12800 X86::LCMPXCHG64, 12801 X86::NOT64r, X86::RAX, 12802 &X86::GR64RegClass); 12803 case X86::ATOMXOR64: 12804 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, 12805 X86::XOR64ri32, X86::MOV64rm, 12806 X86::LCMPXCHG64, 12807 X86::NOT64r, X86::RAX, 12808 &X86::GR64RegClass); 12809 case X86::ATOMNAND64: 12810 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 12811 X86::AND64ri32, X86::MOV64rm, 12812 X86::LCMPXCHG64, 12813 X86::NOT64r, X86::RAX, 12814 &X86::GR64RegClass, true); 12815 case X86::ATOMMIN64: 12816 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); 12817 case X86::ATOMMAX64: 12818 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); 12819 case X86::ATOMUMIN64: 12820 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); 12821 case X86::ATOMUMAX64: 12822 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); 12823 12824 // This group does 64-bit operations on a 32-bit host. 12825 case X86::ATOMAND6432: 12826 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12827 X86::AND32rr, X86::AND32rr, 12828 X86::AND32ri, X86::AND32ri, 12829 false); 12830 case X86::ATOMOR6432: 12831 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12832 X86::OR32rr, X86::OR32rr, 12833 X86::OR32ri, X86::OR32ri, 12834 false); 12835 case X86::ATOMXOR6432: 12836 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12837 X86::XOR32rr, X86::XOR32rr, 12838 X86::XOR32ri, X86::XOR32ri, 12839 false); 12840 case X86::ATOMNAND6432: 12841 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12842 X86::AND32rr, X86::AND32rr, 12843 X86::AND32ri, X86::AND32ri, 12844 true); 12845 case X86::ATOMADD6432: 12846 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12847 X86::ADD32rr, X86::ADC32rr, 12848 X86::ADD32ri, X86::ADC32ri, 12849 false); 12850 case X86::ATOMSUB6432: 12851 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12852 X86::SUB32rr, X86::SBB32rr, 12853 X86::SUB32ri, X86::SBB32ri, 12854 false); 12855 case X86::ATOMSWAP6432: 12856 return EmitAtomicBit6432WithCustomInserter(MI, BB, 12857 X86::MOV32rr, X86::MOV32rr, 12858 X86::MOV32ri, X86::MOV32ri, 12859 false); 12860 case X86::VASTART_SAVE_XMM_REGS: 12861 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB); 12862 12863 case X86::VAARG_64: 12864 return EmitVAARG64WithCustomInserter(MI, BB); 12865 } 12866} 12867 12868//===----------------------------------------------------------------------===// 12869// X86 Optimization Hooks 12870//===----------------------------------------------------------------------===// 12871 12872void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 12873 APInt &KnownZero, 12874 APInt &KnownOne, 12875 const SelectionDAG &DAG, 12876 unsigned Depth) const { 12877 unsigned BitWidth = KnownZero.getBitWidth(); 12878 unsigned Opc = Op.getOpcode(); 12879 assert((Opc >= ISD::BUILTIN_OP_END || 12880 Opc == ISD::INTRINSIC_WO_CHAIN || 12881 Opc == ISD::INTRINSIC_W_CHAIN || 12882 Opc == ISD::INTRINSIC_VOID) && 12883 "Should use MaskedValueIsZero if you don't know whether Op" 12884 " is a target node!"); 12885 12886 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 12887 switch (Opc) { 12888 default: break; 12889 case X86ISD::ADD: 12890 case X86ISD::SUB: 12891 case X86ISD::ADC: 12892 case X86ISD::SBB: 12893 case X86ISD::SMUL: 12894 case X86ISD::UMUL: 12895 case X86ISD::INC: 12896 case X86ISD::DEC: 12897 case X86ISD::OR: 12898 case X86ISD::XOR: 12899 case X86ISD::AND: 12900 // These nodes' second result is a boolean. 12901 if (Op.getResNo() == 0) 12902 break; 12903 // Fallthrough 12904 case X86ISD::SETCC: 12905 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 12906 break; 12907 case ISD::INTRINSIC_WO_CHAIN: { 12908 unsigned IntId = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 12909 unsigned NumLoBits = 0; 12910 switch (IntId) { 12911 default: break; 12912 case Intrinsic::x86_sse_movmsk_ps: 12913 case Intrinsic::x86_avx_movmsk_ps_256: 12914 case Intrinsic::x86_sse2_movmsk_pd: 12915 case Intrinsic::x86_avx_movmsk_pd_256: 12916 case Intrinsic::x86_mmx_pmovmskb: 12917 case Intrinsic::x86_sse2_pmovmskb_128: 12918 case Intrinsic::x86_avx2_pmovmskb: { 12919 // High bits of movmskp{s|d}, pmovmskb are known zero. 12920 switch (IntId) { 12921 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 12922 case Intrinsic::x86_sse_movmsk_ps: NumLoBits = 4; break; 12923 case Intrinsic::x86_avx_movmsk_ps_256: NumLoBits = 8; break; 12924 case Intrinsic::x86_sse2_movmsk_pd: NumLoBits = 2; break; 12925 case Intrinsic::x86_avx_movmsk_pd_256: NumLoBits = 4; break; 12926 case Intrinsic::x86_mmx_pmovmskb: NumLoBits = 8; break; 12927 case Intrinsic::x86_sse2_pmovmskb_128: NumLoBits = 16; break; 12928 case Intrinsic::x86_avx2_pmovmskb: NumLoBits = 32; break; 12929 } 12930 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - NumLoBits); 12931 break; 12932 } 12933 } 12934 break; 12935 } 12936 } 12937} 12938 12939unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op, 12940 unsigned Depth) const { 12941 // SETCC_CARRY sets the dest to ~0 for true or 0 for false. 12942 if (Op.getOpcode() == X86ISD::SETCC_CARRY) 12943 return Op.getValueType().getScalarType().getSizeInBits(); 12944 12945 // Fallback case. 12946 return 1; 12947} 12948 12949/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 12950/// node is a GlobalAddress + offset. 12951bool X86TargetLowering::isGAPlusOffset(SDNode *N, 12952 const GlobalValue* &GA, 12953 int64_t &Offset) const { 12954 if (N->getOpcode() == X86ISD::Wrapper) { 12955 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 12956 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 12957 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset(); 12958 return true; 12959 } 12960 } 12961 return TargetLowering::isGAPlusOffset(N, GA, Offset); 12962} 12963 12964/// isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the 12965/// same as extracting the high 128-bit part of 256-bit vector and then 12966/// inserting the result into the low part of a new 256-bit vector 12967static bool isShuffleHigh128VectorInsertLow(ShuffleVectorSDNode *SVOp) { 12968 EVT VT = SVOp->getValueType(0); 12969 unsigned NumElems = VT.getVectorNumElements(); 12970 12971 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u> 12972 for (unsigned i = 0, j = NumElems/2; i != NumElems/2; ++i, ++j) 12973 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) || 12974 SVOp->getMaskElt(j) >= 0) 12975 return false; 12976 12977 return true; 12978} 12979 12980/// isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the 12981/// same as extracting the low 128-bit part of 256-bit vector and then 12982/// inserting the result into the high part of a new 256-bit vector 12983static bool isShuffleLow128VectorInsertHigh(ShuffleVectorSDNode *SVOp) { 12984 EVT VT = SVOp->getValueType(0); 12985 unsigned NumElems = VT.getVectorNumElements(); 12986 12987 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1> 12988 for (unsigned i = NumElems/2, j = 0; i != NumElems; ++i, ++j) 12989 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) || 12990 SVOp->getMaskElt(j) >= 0) 12991 return false; 12992 12993 return true; 12994} 12995 12996/// PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors. 12997static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG, 12998 TargetLowering::DAGCombinerInfo &DCI, 12999 const X86Subtarget* Subtarget) { 13000 DebugLoc dl = N->getDebugLoc(); 13001 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 13002 SDValue V1 = SVOp->getOperand(0); 13003 SDValue V2 = SVOp->getOperand(1); 13004 EVT VT = SVOp->getValueType(0); 13005 unsigned NumElems = VT.getVectorNumElements(); 13006 13007 if (V1.getOpcode() == ISD::CONCAT_VECTORS && 13008 V2.getOpcode() == ISD::CONCAT_VECTORS) { 13009 // 13010 // 0,0,0,... 13011 // | 13012 // V UNDEF BUILD_VECTOR UNDEF 13013 // \ / \ / 13014 // CONCAT_VECTOR CONCAT_VECTOR 13015 // \ / 13016 // \ / 13017 // RESULT: V + zero extended 13018 // 13019 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR || 13020 V2.getOperand(1).getOpcode() != ISD::UNDEF || 13021 V1.getOperand(1).getOpcode() != ISD::UNDEF) 13022 return SDValue(); 13023 13024 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode())) 13025 return SDValue(); 13026 13027 // To match the shuffle mask, the first half of the mask should 13028 // be exactly the first vector, and all the rest a splat with the 13029 // first element of the second one. 13030 for (unsigned i = 0; i != NumElems/2; ++i) 13031 if (!isUndefOrEqual(SVOp->getMaskElt(i), i) || 13032 !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems)) 13033 return SDValue(); 13034 13035 // If V1 is coming from a vector load then just fold to a VZEXT_LOAD. 13036 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(V1.getOperand(0))) { 13037 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other); 13038 SDValue Ops[] = { Ld->getChain(), Ld->getBasePtr() }; 13039 SDValue ResNode = 13040 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, 2, 13041 Ld->getMemoryVT(), 13042 Ld->getPointerInfo(), 13043 Ld->getAlignment(), 13044 false/*isVolatile*/, true/*ReadMem*/, 13045 false/*WriteMem*/); 13046 return DAG.getNode(ISD::BITCAST, dl, VT, ResNode); 13047 } 13048 13049 // Emit a zeroed vector and insert the desired subvector on its 13050 // first half. 13051 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl); 13052 SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 0, DAG, dl); 13053 return DCI.CombineTo(N, InsV); 13054 } 13055 13056 //===--------------------------------------------------------------------===// 13057 // Combine some shuffles into subvector extracts and inserts: 13058 // 13059 13060 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u> 13061 if (isShuffleHigh128VectorInsertLow(SVOp)) { 13062 SDValue V = Extract128BitVector(V1, NumElems/2, DAG, dl); 13063 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, 0, DAG, dl); 13064 return DCI.CombineTo(N, InsV); 13065 } 13066 13067 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1> 13068 if (isShuffleLow128VectorInsertHigh(SVOp)) { 13069 SDValue V = Extract128BitVector(V1, 0, DAG, dl); 13070 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, NumElems/2, DAG, dl); 13071 return DCI.CombineTo(N, InsV); 13072 } 13073 13074 return SDValue(); 13075} 13076 13077/// PerformShuffleCombine - Performs several different shuffle combines. 13078static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 13079 TargetLowering::DAGCombinerInfo &DCI, 13080 const X86Subtarget *Subtarget) { 13081 DebugLoc dl = N->getDebugLoc(); 13082 EVT VT = N->getValueType(0); 13083 13084 // Don't create instructions with illegal types after legalize types has run. 13085 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 13086 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType())) 13087 return SDValue(); 13088 13089 // Combine 256-bit vector shuffles. This is only profitable when in AVX mode 13090 if (Subtarget->hasAVX() && VT.getSizeInBits() == 256 && 13091 N->getOpcode() == ISD::VECTOR_SHUFFLE) 13092 return PerformShuffleCombine256(N, DAG, DCI, Subtarget); 13093 13094 // Only handle 128 wide vector from here on. 13095 if (VT.getSizeInBits() != 128) 13096 return SDValue(); 13097 13098 // Combine a vector_shuffle that is equal to build_vector load1, load2, load3, 13099 // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are 13100 // consecutive, non-overlapping, and in the right order. 13101 SmallVector<SDValue, 16> Elts; 13102 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) 13103 Elts.push_back(getShuffleScalarElt(N, i, DAG, 0)); 13104 13105 return EltsFromConsecutiveLoads(VT, Elts, dl, DAG); 13106} 13107 13108 13109/// DCI, PerformTruncateCombine - Converts truncate operation to 13110/// a sequence of vector shuffle operations. 13111/// It is possible when we truncate 256-bit vector to 128-bit vector 13112 13113SDValue X86TargetLowering::PerformTruncateCombine(SDNode *N, SelectionDAG &DAG, 13114 DAGCombinerInfo &DCI) const { 13115 if (!DCI.isBeforeLegalizeOps()) 13116 return SDValue(); 13117 13118 if (!Subtarget->hasAVX()) 13119 return SDValue(); 13120 13121 EVT VT = N->getValueType(0); 13122 SDValue Op = N->getOperand(0); 13123 EVT OpVT = Op.getValueType(); 13124 DebugLoc dl = N->getDebugLoc(); 13125 13126 if ((VT == MVT::v4i32) && (OpVT == MVT::v4i64)) { 13127 13128 if (Subtarget->hasAVX2()) { 13129 // AVX2: v4i64 -> v4i32 13130 13131 // VPERMD 13132 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1}; 13133 13134 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v8i32, Op); 13135 Op = DAG.getVectorShuffle(MVT::v8i32, dl, Op, DAG.getUNDEF(MVT::v8i32), 13136 ShufMask); 13137 13138 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op, 13139 DAG.getIntPtrConstant(0)); 13140 } 13141 13142 // AVX: v4i64 -> v4i32 13143 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Op, 13144 DAG.getIntPtrConstant(0)); 13145 13146 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Op, 13147 DAG.getIntPtrConstant(2)); 13148 13149 OpLo = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, OpLo); 13150 OpHi = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, OpHi); 13151 13152 // PSHUFD 13153 static const int ShufMask1[] = {0, 2, 0, 0}; 13154 13155 OpLo = DAG.getVectorShuffle(VT, dl, OpLo, DAG.getUNDEF(VT), ShufMask1); 13156 OpHi = DAG.getVectorShuffle(VT, dl, OpHi, DAG.getUNDEF(VT), ShufMask1); 13157 13158 // MOVLHPS 13159 static const int ShufMask2[] = {0, 1, 4, 5}; 13160 13161 return DAG.getVectorShuffle(VT, dl, OpLo, OpHi, ShufMask2); 13162 } 13163 13164 if ((VT == MVT::v8i16) && (OpVT == MVT::v8i32)) { 13165 13166 if (Subtarget->hasAVX2()) { 13167 // AVX2: v8i32 -> v8i16 13168 13169 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v32i8, Op); 13170 13171 // PSHUFB 13172 SmallVector<SDValue,32> pshufbMask; 13173 for (unsigned i = 0; i < 2; ++i) { 13174 pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8)); 13175 pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8)); 13176 pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8)); 13177 pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8)); 13178 pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8)); 13179 pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8)); 13180 pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8)); 13181 pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8)); 13182 for (unsigned j = 0; j < 8; ++j) 13183 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 13184 } 13185 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v32i8, 13186 &pshufbMask[0], 32); 13187 Op = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v32i8, Op, BV); 13188 13189 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4i64, Op); 13190 13191 static const int ShufMask[] = {0, 2, -1, -1}; 13192 Op = DAG.getVectorShuffle(MVT::v4i64, dl, Op, DAG.getUNDEF(MVT::v4i64), 13193 &ShufMask[0]); 13194 13195 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Op, 13196 DAG.getIntPtrConstant(0)); 13197 13198 return DAG.getNode(ISD::BITCAST, dl, VT, Op); 13199 } 13200 13201 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i32, Op, 13202 DAG.getIntPtrConstant(0)); 13203 13204 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i32, Op, 13205 DAG.getIntPtrConstant(4)); 13206 13207 OpLo = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLo); 13208 OpHi = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpHi); 13209 13210 // PSHUFB 13211 static const int ShufMask1[] = {0, 1, 4, 5, 8, 9, 12, 13, 13212 -1, -1, -1, -1, -1, -1, -1, -1}; 13213 13214 OpLo = DAG.getVectorShuffle(MVT::v16i8, dl, OpLo, DAG.getUNDEF(MVT::v16i8), 13215 ShufMask1); 13216 OpHi = DAG.getVectorShuffle(MVT::v16i8, dl, OpHi, DAG.getUNDEF(MVT::v16i8), 13217 ShufMask1); 13218 13219 OpLo = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, OpLo); 13220 OpHi = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, OpHi); 13221 13222 // MOVLHPS 13223 static const int ShufMask2[] = {0, 1, 4, 5}; 13224 13225 SDValue res = DAG.getVectorShuffle(MVT::v4i32, dl, OpLo, OpHi, ShufMask2); 13226 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, res); 13227 } 13228 13229 return SDValue(); 13230} 13231 13232/// XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target 13233/// specific shuffle of a load can be folded into a single element load. 13234/// Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but 13235/// shuffles have been customed lowered so we need to handle those here. 13236static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG, 13237 TargetLowering::DAGCombinerInfo &DCI) { 13238 if (DCI.isBeforeLegalizeOps()) 13239 return SDValue(); 13240 13241 SDValue InVec = N->getOperand(0); 13242 SDValue EltNo = N->getOperand(1); 13243 13244 if (!isa<ConstantSDNode>(EltNo)) 13245 return SDValue(); 13246 13247 EVT VT = InVec.getValueType(); 13248 13249 bool HasShuffleIntoBitcast = false; 13250 if (InVec.getOpcode() == ISD::BITCAST) { 13251 // Don't duplicate a load with other uses. 13252 if (!InVec.hasOneUse()) 13253 return SDValue(); 13254 EVT BCVT = InVec.getOperand(0).getValueType(); 13255 if (BCVT.getVectorNumElements() != VT.getVectorNumElements()) 13256 return SDValue(); 13257 InVec = InVec.getOperand(0); 13258 HasShuffleIntoBitcast = true; 13259 } 13260 13261 if (!isTargetShuffle(InVec.getOpcode())) 13262 return SDValue(); 13263 13264 // Don't duplicate a load with other uses. 13265 if (!InVec.hasOneUse()) 13266 return SDValue(); 13267 13268 SmallVector<int, 16> ShuffleMask; 13269 bool UnaryShuffle; 13270 if (!getTargetShuffleMask(InVec.getNode(), VT.getSimpleVT(), ShuffleMask, 13271 UnaryShuffle)) 13272 return SDValue(); 13273 13274 // Select the input vector, guarding against out of range extract vector. 13275 unsigned NumElems = VT.getVectorNumElements(); 13276 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 13277 int Idx = (Elt > (int)NumElems) ? -1 : ShuffleMask[Elt]; 13278 SDValue LdNode = (Idx < (int)NumElems) ? InVec.getOperand(0) 13279 : InVec.getOperand(1); 13280 13281 // If inputs to shuffle are the same for both ops, then allow 2 uses 13282 unsigned AllowedUses = InVec.getOperand(0) == InVec.getOperand(1) ? 2 : 1; 13283 13284 if (LdNode.getOpcode() == ISD::BITCAST) { 13285 // Don't duplicate a load with other uses. 13286 if (!LdNode.getNode()->hasNUsesOfValue(AllowedUses, 0)) 13287 return SDValue(); 13288 13289 AllowedUses = 1; // only allow 1 load use if we have a bitcast 13290 LdNode = LdNode.getOperand(0); 13291 } 13292 13293 if (!ISD::isNormalLoad(LdNode.getNode())) 13294 return SDValue(); 13295 13296 LoadSDNode *LN0 = cast<LoadSDNode>(LdNode); 13297 13298 if (!LN0 ||!LN0->hasNUsesOfValue(AllowedUses, 0) || LN0->isVolatile()) 13299 return SDValue(); 13300 13301 if (HasShuffleIntoBitcast) { 13302 // If there's a bitcast before the shuffle, check if the load type and 13303 // alignment is valid. 13304 unsigned Align = LN0->getAlignment(); 13305 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 13306 unsigned NewAlign = TLI.getTargetData()-> 13307 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext())); 13308 13309 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VT)) 13310 return SDValue(); 13311 } 13312 13313 // All checks match so transform back to vector_shuffle so that DAG combiner 13314 // can finish the job 13315 DebugLoc dl = N->getDebugLoc(); 13316 13317 // Create shuffle node taking into account the case that its a unary shuffle 13318 SDValue Shuffle = (UnaryShuffle) ? DAG.getUNDEF(VT) : InVec.getOperand(1); 13319 Shuffle = DAG.getVectorShuffle(InVec.getValueType(), dl, 13320 InVec.getOperand(0), Shuffle, 13321 &ShuffleMask[0]); 13322 Shuffle = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); 13323 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle, 13324 EltNo); 13325} 13326 13327/// PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index 13328/// generation and convert it from being a bunch of shuffles and extracts 13329/// to a simple store and scalar loads to extract the elements. 13330static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, 13331 TargetLowering::DAGCombinerInfo &DCI) { 13332 SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI); 13333 if (NewOp.getNode()) 13334 return NewOp; 13335 13336 SDValue InputVector = N->getOperand(0); 13337 13338 // Only operate on vectors of 4 elements, where the alternative shuffling 13339 // gets to be more expensive. 13340 if (InputVector.getValueType() != MVT::v4i32) 13341 return SDValue(); 13342 13343 // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a 13344 // single use which is a sign-extend or zero-extend, and all elements are 13345 // used. 13346 SmallVector<SDNode *, 4> Uses; 13347 unsigned ExtractedElements = 0; 13348 for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(), 13349 UE = InputVector.getNode()->use_end(); UI != UE; ++UI) { 13350 if (UI.getUse().getResNo() != InputVector.getResNo()) 13351 return SDValue(); 13352 13353 SDNode *Extract = *UI; 13354 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 13355 return SDValue(); 13356 13357 if (Extract->getValueType(0) != MVT::i32) 13358 return SDValue(); 13359 if (!Extract->hasOneUse()) 13360 return SDValue(); 13361 if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND && 13362 Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND) 13363 return SDValue(); 13364 if (!isa<ConstantSDNode>(Extract->getOperand(1))) 13365 return SDValue(); 13366 13367 // Record which element was extracted. 13368 ExtractedElements |= 13369 1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue(); 13370 13371 Uses.push_back(Extract); 13372 } 13373 13374 // If not all the elements were used, this may not be worthwhile. 13375 if (ExtractedElements != 15) 13376 return SDValue(); 13377 13378 // Ok, we've now decided to do the transformation. 13379 DebugLoc dl = InputVector.getDebugLoc(); 13380 13381 // Store the value to a temporary stack slot. 13382 SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType()); 13383 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr, 13384 MachinePointerInfo(), false, false, 0); 13385 13386 // Replace each use (extract) with a load of the appropriate element. 13387 for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(), 13388 UE = Uses.end(); UI != UE; ++UI) { 13389 SDNode *Extract = *UI; 13390 13391 // cOMpute the element's address. 13392 SDValue Idx = Extract->getOperand(1); 13393 unsigned EltSize = 13394 InputVector.getValueType().getVectorElementType().getSizeInBits()/8; 13395 uint64_t Offset = EltSize * cast<ConstantSDNode>(Idx)->getZExtValue(); 13396 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 13397 SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy()); 13398 13399 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), 13400 StackPtr, OffsetVal); 13401 13402 // Load the scalar. 13403 SDValue LoadScalar = DAG.getLoad(Extract->getValueType(0), dl, Ch, 13404 ScalarAddr, MachinePointerInfo(), 13405 false, false, false, 0); 13406 13407 // Replace the exact with the load. 13408 DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), LoadScalar); 13409 } 13410 13411 // The replacement was made in place; don't return anything. 13412 return SDValue(); 13413} 13414 13415/// PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT 13416/// nodes. 13417static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 13418 TargetLowering::DAGCombinerInfo &DCI, 13419 const X86Subtarget *Subtarget) { 13420 13421 13422 DebugLoc DL = N->getDebugLoc(); 13423 SDValue Cond = N->getOperand(0); 13424 // Get the LHS/RHS of the select. 13425 SDValue LHS = N->getOperand(1); 13426 SDValue RHS = N->getOperand(2); 13427 EVT VT = LHS.getValueType(); 13428 13429 // If we have SSE[12] support, try to form min/max nodes. SSE min/max 13430 // instructions match the semantics of the common C idiom x<y?x:y but not 13431 // x<=y?x:y, because of how they handle negative zero (which can be 13432 // ignored in unsafe-math mode). 13433 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() && 13434 VT != MVT::f80 && DAG.getTargetLoweringInfo().isTypeLegal(VT) && 13435 (Subtarget->hasSSE2() || 13436 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) { 13437 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 13438 13439 unsigned Opcode = 0; 13440 // Check for x CC y ? x : y. 13441 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) && 13442 DAG.isEqualTo(RHS, Cond.getOperand(1))) { 13443 switch (CC) { 13444 default: break; 13445 case ISD::SETULT: 13446 // Converting this to a min would handle NaNs incorrectly, and swapping 13447 // the operands would cause it to handle comparisons between positive 13448 // and negative zero incorrectly. 13449 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { 13450 if (!DAG.getTarget().Options.UnsafeFPMath && 13451 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) 13452 break; 13453 std::swap(LHS, RHS); 13454 } 13455 Opcode = X86ISD::FMIN; 13456 break; 13457 case ISD::SETOLE: 13458 // Converting this to a min would handle comparisons between positive 13459 // and negative zero incorrectly. 13460 if (!DAG.getTarget().Options.UnsafeFPMath && 13461 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) 13462 break; 13463 Opcode = X86ISD::FMIN; 13464 break; 13465 case ISD::SETULE: 13466 // Converting this to a min would handle both negative zeros and NaNs 13467 // incorrectly, but we can swap the operands to fix both. 13468 std::swap(LHS, RHS); 13469 case ISD::SETOLT: 13470 case ISD::SETLT: 13471 case ISD::SETLE: 13472 Opcode = X86ISD::FMIN; 13473 break; 13474 13475 case ISD::SETOGE: 13476 // Converting this to a max would handle comparisons between positive 13477 // and negative zero incorrectly. 13478 if (!DAG.getTarget().Options.UnsafeFPMath && 13479 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) 13480 break; 13481 Opcode = X86ISD::FMAX; 13482 break; 13483 case ISD::SETUGT: 13484 // Converting this to a max would handle NaNs incorrectly, and swapping 13485 // the operands would cause it to handle comparisons between positive 13486 // and negative zero incorrectly. 13487 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { 13488 if (!DAG.getTarget().Options.UnsafeFPMath && 13489 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) 13490 break; 13491 std::swap(LHS, RHS); 13492 } 13493 Opcode = X86ISD::FMAX; 13494 break; 13495 case ISD::SETUGE: 13496 // Converting this to a max would handle both negative zeros and NaNs 13497 // incorrectly, but we can swap the operands to fix both. 13498 std::swap(LHS, RHS); 13499 case ISD::SETOGT: 13500 case ISD::SETGT: 13501 case ISD::SETGE: 13502 Opcode = X86ISD::FMAX; 13503 break; 13504 } 13505 // Check for x CC y ? y : x -- a min/max with reversed arms. 13506 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) && 13507 DAG.isEqualTo(RHS, Cond.getOperand(0))) { 13508 switch (CC) { 13509 default: break; 13510 case ISD::SETOGE: 13511 // Converting this to a min would handle comparisons between positive 13512 // and negative zero incorrectly, and swapping the operands would 13513 // cause it to handle NaNs incorrectly. 13514 if (!DAG.getTarget().Options.UnsafeFPMath && 13515 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) { 13516 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 13517 break; 13518 std::swap(LHS, RHS); 13519 } 13520 Opcode = X86ISD::FMIN; 13521 break; 13522 case ISD::SETUGT: 13523 // Converting this to a min would handle NaNs incorrectly. 13524 if (!DAG.getTarget().Options.UnsafeFPMath && 13525 (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))) 13526 break; 13527 Opcode = X86ISD::FMIN; 13528 break; 13529 case ISD::SETUGE: 13530 // Converting this to a min would handle both negative zeros and NaNs 13531 // incorrectly, but we can swap the operands to fix both. 13532 std::swap(LHS, RHS); 13533 case ISD::SETOGT: 13534 case ISD::SETGT: 13535 case ISD::SETGE: 13536 Opcode = X86ISD::FMIN; 13537 break; 13538 13539 case ISD::SETULT: 13540 // Converting this to a max would handle NaNs incorrectly. 13541 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 13542 break; 13543 Opcode = X86ISD::FMAX; 13544 break; 13545 case ISD::SETOLE: 13546 // Converting this to a max would handle comparisons between positive 13547 // and negative zero incorrectly, and swapping the operands would 13548 // cause it to handle NaNs incorrectly. 13549 if (!DAG.getTarget().Options.UnsafeFPMath && 13550 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) { 13551 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) 13552 break; 13553 std::swap(LHS, RHS); 13554 } 13555 Opcode = X86ISD::FMAX; 13556 break; 13557 case ISD::SETULE: 13558 // Converting this to a max would handle both negative zeros and NaNs 13559 // incorrectly, but we can swap the operands to fix both. 13560 std::swap(LHS, RHS); 13561 case ISD::SETOLT: 13562 case ISD::SETLT: 13563 case ISD::SETLE: 13564 Opcode = X86ISD::FMAX; 13565 break; 13566 } 13567 } 13568 13569 if (Opcode) 13570 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); 13571 } 13572 13573 // If this is a select between two integer constants, try to do some 13574 // optimizations. 13575 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) { 13576 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS)) 13577 // Don't do this for crazy integer types. 13578 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) { 13579 // If this is efficiently invertible, canonicalize the LHSC/RHSC values 13580 // so that TrueC (the true value) is larger than FalseC. 13581 bool NeedsCondInvert = false; 13582 13583 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) && 13584 // Efficiently invertible. 13585 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible. 13586 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible. 13587 isa<ConstantSDNode>(Cond.getOperand(1))))) { 13588 NeedsCondInvert = true; 13589 std::swap(TrueC, FalseC); 13590 } 13591 13592 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0. 13593 if (FalseC->getAPIntValue() == 0 && 13594 TrueC->getAPIntValue().isPowerOf2()) { 13595 if (NeedsCondInvert) // Invert the condition if needed. 13596 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 13597 DAG.getConstant(1, Cond.getValueType())); 13598 13599 // Zero extend the condition if needed. 13600 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond); 13601 13602 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 13603 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond, 13604 DAG.getConstant(ShAmt, MVT::i8)); 13605 } 13606 13607 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. 13608 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 13609 if (NeedsCondInvert) // Invert the condition if needed. 13610 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 13611 DAG.getConstant(1, Cond.getValueType())); 13612 13613 // Zero extend the condition if needed. 13614 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 13615 FalseC->getValueType(0), Cond); 13616 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 13617 SDValue(FalseC, 0)); 13618 } 13619 13620 // Optimize cases that will turn into an LEA instruction. This requires 13621 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 13622 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 13623 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 13624 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 13625 13626 bool isFastMultiplier = false; 13627 if (Diff < 10) { 13628 switch ((unsigned char)Diff) { 13629 default: break; 13630 case 1: // result = add base, cond 13631 case 2: // result = lea base( , cond*2) 13632 case 3: // result = lea base(cond, cond*2) 13633 case 4: // result = lea base( , cond*4) 13634 case 5: // result = lea base(cond, cond*4) 13635 case 8: // result = lea base( , cond*8) 13636 case 9: // result = lea base(cond, cond*8) 13637 isFastMultiplier = true; 13638 break; 13639 } 13640 } 13641 13642 if (isFastMultiplier) { 13643 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 13644 if (NeedsCondInvert) // Invert the condition if needed. 13645 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 13646 DAG.getConstant(1, Cond.getValueType())); 13647 13648 // Zero extend the condition if needed. 13649 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 13650 Cond); 13651 // Scale the condition by the difference. 13652 if (Diff != 1) 13653 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 13654 DAG.getConstant(Diff, Cond.getValueType())); 13655 13656 // Add the base if non-zero. 13657 if (FalseC->getAPIntValue() != 0) 13658 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 13659 SDValue(FalseC, 0)); 13660 return Cond; 13661 } 13662 } 13663 } 13664 } 13665 13666 // Canonicalize max and min: 13667 // (x > y) ? x : y -> (x >= y) ? x : y 13668 // (x < y) ? x : y -> (x <= y) ? x : y 13669 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates 13670 // the need for an extra compare 13671 // against zero. e.g. 13672 // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0 13673 // subl %esi, %edi 13674 // testl %edi, %edi 13675 // movl $0, %eax 13676 // cmovgl %edi, %eax 13677 // => 13678 // xorl %eax, %eax 13679 // subl %esi, $edi 13680 // cmovsl %eax, %edi 13681 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC && 13682 DAG.isEqualTo(LHS, Cond.getOperand(0)) && 13683 DAG.isEqualTo(RHS, Cond.getOperand(1))) { 13684 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 13685 switch (CC) { 13686 default: break; 13687 case ISD::SETLT: 13688 case ISD::SETGT: { 13689 ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE; 13690 Cond = DAG.getSetCC(Cond.getDebugLoc(), Cond.getValueType(), 13691 Cond.getOperand(0), Cond.getOperand(1), NewCC); 13692 return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS); 13693 } 13694 } 13695 } 13696 13697 // If we know that this node is legal then we know that it is going to be 13698 // matched by one of the SSE/AVX BLEND instructions. These instructions only 13699 // depend on the highest bit in each word. Try to use SimplifyDemandedBits 13700 // to simplify previous instructions. 13701 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 13702 if (N->getOpcode() == ISD::VSELECT && DCI.isBeforeLegalizeOps() && 13703 !DCI.isBeforeLegalize() && 13704 TLI.isOperationLegal(ISD::VSELECT, VT)) { 13705 unsigned BitWidth = Cond.getValueType().getScalarType().getSizeInBits(); 13706 assert(BitWidth >= 8 && BitWidth <= 64 && "Invalid mask size"); 13707 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 1); 13708 13709 APInt KnownZero, KnownOne; 13710 TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(), 13711 DCI.isBeforeLegalizeOps()); 13712 if (TLO.ShrinkDemandedConstant(Cond, DemandedMask) || 13713 TLI.SimplifyDemandedBits(Cond, DemandedMask, KnownZero, KnownOne, TLO)) 13714 DCI.CommitTargetLoweringOpt(TLO); 13715 } 13716 13717 return SDValue(); 13718} 13719 13720/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL] 13721static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG, 13722 TargetLowering::DAGCombinerInfo &DCI) { 13723 DebugLoc DL = N->getDebugLoc(); 13724 13725 // If the flag operand isn't dead, don't touch this CMOV. 13726 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty()) 13727 return SDValue(); 13728 13729 SDValue FalseOp = N->getOperand(0); 13730 SDValue TrueOp = N->getOperand(1); 13731 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2); 13732 SDValue Cond = N->getOperand(3); 13733 if (CC == X86::COND_E || CC == X86::COND_NE) { 13734 switch (Cond.getOpcode()) { 13735 default: break; 13736 case X86ISD::BSR: 13737 case X86ISD::BSF: 13738 // If operand of BSR / BSF are proven never zero, then ZF cannot be set. 13739 if (DAG.isKnownNeverZero(Cond.getOperand(0))) 13740 return (CC == X86::COND_E) ? FalseOp : TrueOp; 13741 } 13742 } 13743 13744 // If this is a select between two integer constants, try to do some 13745 // optimizations. Note that the operands are ordered the opposite of SELECT 13746 // operands. 13747 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) { 13748 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) { 13749 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is 13750 // larger than FalseC (the false value). 13751 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) { 13752 CC = X86::GetOppositeBranchCondition(CC); 13753 std::swap(TrueC, FalseC); 13754 } 13755 13756 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0. 13757 // This is efficient for any integer data type (including i8/i16) and 13758 // shift amount. 13759 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) { 13760 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 13761 DAG.getConstant(CC, MVT::i8), Cond); 13762 13763 // Zero extend the condition if needed. 13764 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); 13765 13766 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 13767 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, 13768 DAG.getConstant(ShAmt, MVT::i8)); 13769 if (N->getNumValues() == 2) // Dead flag value? 13770 return DCI.CombineTo(N, Cond, SDValue()); 13771 return Cond; 13772 } 13773 13774 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient 13775 // for any integer data type, including i8/i16. 13776 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 13777 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 13778 DAG.getConstant(CC, MVT::i8), Cond); 13779 13780 // Zero extend the condition if needed. 13781 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 13782 FalseC->getValueType(0), Cond); 13783 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 13784 SDValue(FalseC, 0)); 13785 13786 if (N->getNumValues() == 2) // Dead flag value? 13787 return DCI.CombineTo(N, Cond, SDValue()); 13788 return Cond; 13789 } 13790 13791 // Optimize cases that will turn into an LEA instruction. This requires 13792 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 13793 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 13794 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 13795 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 13796 13797 bool isFastMultiplier = false; 13798 if (Diff < 10) { 13799 switch ((unsigned char)Diff) { 13800 default: break; 13801 case 1: // result = add base, cond 13802 case 2: // result = lea base( , cond*2) 13803 case 3: // result = lea base(cond, cond*2) 13804 case 4: // result = lea base( , cond*4) 13805 case 5: // result = lea base(cond, cond*4) 13806 case 8: // result = lea base( , cond*8) 13807 case 9: // result = lea base(cond, cond*8) 13808 isFastMultiplier = true; 13809 break; 13810 } 13811 } 13812 13813 if (isFastMultiplier) { 13814 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 13815 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 13816 DAG.getConstant(CC, MVT::i8), Cond); 13817 // Zero extend the condition if needed. 13818 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 13819 Cond); 13820 // Scale the condition by the difference. 13821 if (Diff != 1) 13822 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 13823 DAG.getConstant(Diff, Cond.getValueType())); 13824 13825 // Add the base if non-zero. 13826 if (FalseC->getAPIntValue() != 0) 13827 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 13828 SDValue(FalseC, 0)); 13829 if (N->getNumValues() == 2) // Dead flag value? 13830 return DCI.CombineTo(N, Cond, SDValue()); 13831 return Cond; 13832 } 13833 } 13834 } 13835 } 13836 return SDValue(); 13837} 13838 13839 13840/// PerformMulCombine - Optimize a single multiply with constant into two 13841/// in order to implement it with two cheaper instructions, e.g. 13842/// LEA + SHL, LEA + LEA. 13843static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG, 13844 TargetLowering::DAGCombinerInfo &DCI) { 13845 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 13846 return SDValue(); 13847 13848 EVT VT = N->getValueType(0); 13849 if (VT != MVT::i64) 13850 return SDValue(); 13851 13852 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 13853 if (!C) 13854 return SDValue(); 13855 uint64_t MulAmt = C->getZExtValue(); 13856 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9) 13857 return SDValue(); 13858 13859 uint64_t MulAmt1 = 0; 13860 uint64_t MulAmt2 = 0; 13861 if ((MulAmt % 9) == 0) { 13862 MulAmt1 = 9; 13863 MulAmt2 = MulAmt / 9; 13864 } else if ((MulAmt % 5) == 0) { 13865 MulAmt1 = 5; 13866 MulAmt2 = MulAmt / 5; 13867 } else if ((MulAmt % 3) == 0) { 13868 MulAmt1 = 3; 13869 MulAmt2 = MulAmt / 3; 13870 } 13871 if (MulAmt2 && 13872 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){ 13873 DebugLoc DL = N->getDebugLoc(); 13874 13875 if (isPowerOf2_64(MulAmt2) && 13876 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD)) 13877 // If second multiplifer is pow2, issue it first. We want the multiply by 13878 // 3, 5, or 9 to be folded into the addressing mode unless the lone use 13879 // is an add. 13880 std::swap(MulAmt1, MulAmt2); 13881 13882 SDValue NewMul; 13883 if (isPowerOf2_64(MulAmt1)) 13884 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 13885 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); 13886 else 13887 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), 13888 DAG.getConstant(MulAmt1, VT)); 13889 13890 if (isPowerOf2_64(MulAmt2)) 13891 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, 13892 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); 13893 else 13894 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, 13895 DAG.getConstant(MulAmt2, VT)); 13896 13897 // Do not add new nodes to DAG combiner worklist. 13898 DCI.CombineTo(N, NewMul, false); 13899 } 13900 return SDValue(); 13901} 13902 13903static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) { 13904 SDValue N0 = N->getOperand(0); 13905 SDValue N1 = N->getOperand(1); 13906 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 13907 EVT VT = N0.getValueType(); 13908 13909 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2)) 13910 // since the result of setcc_c is all zero's or all ones. 13911 if (VT.isInteger() && !VT.isVector() && 13912 N1C && N0.getOpcode() == ISD::AND && 13913 N0.getOperand(1).getOpcode() == ISD::Constant) { 13914 SDValue N00 = N0.getOperand(0); 13915 if (N00.getOpcode() == X86ISD::SETCC_CARRY || 13916 ((N00.getOpcode() == ISD::ANY_EXTEND || 13917 N00.getOpcode() == ISD::ZERO_EXTEND) && 13918 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) { 13919 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 13920 APInt ShAmt = N1C->getAPIntValue(); 13921 Mask = Mask.shl(ShAmt); 13922 if (Mask != 0) 13923 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, 13924 N00, DAG.getConstant(Mask, VT)); 13925 } 13926 } 13927 13928 13929 // Hardware support for vector shifts is sparse which makes us scalarize the 13930 // vector operations in many cases. Also, on sandybridge ADD is faster than 13931 // shl. 13932 // (shl V, 1) -> add V,V 13933 if (isSplatVector(N1.getNode())) { 13934 assert(N0.getValueType().isVector() && "Invalid vector shift type"); 13935 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(0)); 13936 // We shift all of the values by one. In many cases we do not have 13937 // hardware support for this operation. This is better expressed as an ADD 13938 // of two values. 13939 if (N1C && (1 == N1C->getZExtValue())) { 13940 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, N0); 13941 } 13942 } 13943 13944 return SDValue(); 13945} 13946 13947/// PerformShiftCombine - Transforms vector shift nodes to use vector shifts 13948/// when possible. 13949static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG, 13950 TargetLowering::DAGCombinerInfo &DCI, 13951 const X86Subtarget *Subtarget) { 13952 EVT VT = N->getValueType(0); 13953 if (N->getOpcode() == ISD::SHL) { 13954 SDValue V = PerformSHLCombine(N, DAG); 13955 if (V.getNode()) return V; 13956 } 13957 13958 // On X86 with SSE2 support, we can transform this to a vector shift if 13959 // all elements are shifted by the same amount. We can't do this in legalize 13960 // because the a constant vector is typically transformed to a constant pool 13961 // so we have no knowledge of the shift amount. 13962 if (!Subtarget->hasSSE2()) 13963 return SDValue(); 13964 13965 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 && 13966 (!Subtarget->hasAVX2() || 13967 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16))) 13968 return SDValue(); 13969 13970 SDValue ShAmtOp = N->getOperand(1); 13971 EVT EltVT = VT.getVectorElementType(); 13972 DebugLoc DL = N->getDebugLoc(); 13973 SDValue BaseShAmt = SDValue(); 13974 if (ShAmtOp.getOpcode() == ISD::BUILD_VECTOR) { 13975 unsigned NumElts = VT.getVectorNumElements(); 13976 unsigned i = 0; 13977 for (; i != NumElts; ++i) { 13978 SDValue Arg = ShAmtOp.getOperand(i); 13979 if (Arg.getOpcode() == ISD::UNDEF) continue; 13980 BaseShAmt = Arg; 13981 break; 13982 } 13983 // Handle the case where the build_vector is all undef 13984 // FIXME: Should DAG allow this? 13985 if (i == NumElts) 13986 return SDValue(); 13987 13988 for (; i != NumElts; ++i) { 13989 SDValue Arg = ShAmtOp.getOperand(i); 13990 if (Arg.getOpcode() == ISD::UNDEF) continue; 13991 if (Arg != BaseShAmt) { 13992 return SDValue(); 13993 } 13994 } 13995 } else if (ShAmtOp.getOpcode() == ISD::VECTOR_SHUFFLE && 13996 cast<ShuffleVectorSDNode>(ShAmtOp)->isSplat()) { 13997 SDValue InVec = ShAmtOp.getOperand(0); 13998 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 13999 unsigned NumElts = InVec.getValueType().getVectorNumElements(); 14000 unsigned i = 0; 14001 for (; i != NumElts; ++i) { 14002 SDValue Arg = InVec.getOperand(i); 14003 if (Arg.getOpcode() == ISD::UNDEF) continue; 14004 BaseShAmt = Arg; 14005 break; 14006 } 14007 } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) { 14008 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(InVec.getOperand(2))) { 14009 unsigned SplatIdx= cast<ShuffleVectorSDNode>(ShAmtOp)->getSplatIndex(); 14010 if (C->getZExtValue() == SplatIdx) 14011 BaseShAmt = InVec.getOperand(1); 14012 } 14013 } 14014 if (BaseShAmt.getNode() == 0) { 14015 // Don't create instructions with illegal types after legalize 14016 // types has run. 14017 if (!DAG.getTargetLoweringInfo().isTypeLegal(EltVT) && 14018 !DCI.isBeforeLegalize()) 14019 return SDValue(); 14020 14021 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp, 14022 DAG.getIntPtrConstant(0)); 14023 } 14024 } else 14025 return SDValue(); 14026 14027 // The shift amount is an i32. 14028 if (EltVT.bitsGT(MVT::i32)) 14029 BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt); 14030 else if (EltVT.bitsLT(MVT::i32)) 14031 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseShAmt); 14032 14033 // The shift amount is identical so we can do a vector shift. 14034 SDValue ValOp = N->getOperand(0); 14035 switch (N->getOpcode()) { 14036 default: 14037 llvm_unreachable("Unknown shift opcode!"); 14038 case ISD::SHL: 14039 switch (VT.getSimpleVT().SimpleTy) { 14040 default: return SDValue(); 14041 case MVT::v2i64: 14042 case MVT::v4i32: 14043 case MVT::v8i16: 14044 case MVT::v4i64: 14045 case MVT::v8i32: 14046 case MVT::v16i16: 14047 return getTargetVShiftNode(X86ISD::VSHLI, DL, VT, ValOp, BaseShAmt, DAG); 14048 } 14049 case ISD::SRA: 14050 switch (VT.getSimpleVT().SimpleTy) { 14051 default: return SDValue(); 14052 case MVT::v4i32: 14053 case MVT::v8i16: 14054 case MVT::v8i32: 14055 case MVT::v16i16: 14056 return getTargetVShiftNode(X86ISD::VSRAI, DL, VT, ValOp, BaseShAmt, DAG); 14057 } 14058 case ISD::SRL: 14059 switch (VT.getSimpleVT().SimpleTy) { 14060 default: return SDValue(); 14061 case MVT::v2i64: 14062 case MVT::v4i32: 14063 case MVT::v8i16: 14064 case MVT::v4i64: 14065 case MVT::v8i32: 14066 case MVT::v16i16: 14067 return getTargetVShiftNode(X86ISD::VSRLI, DL, VT, ValOp, BaseShAmt, DAG); 14068 } 14069 } 14070} 14071 14072 14073// CMPEQCombine - Recognize the distinctive (AND (setcc ...) (setcc ..)) 14074// where both setccs reference the same FP CMP, and rewrite for CMPEQSS 14075// and friends. Likewise for OR -> CMPNEQSS. 14076static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG, 14077 TargetLowering::DAGCombinerInfo &DCI, 14078 const X86Subtarget *Subtarget) { 14079 unsigned opcode; 14080 14081 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but 14082 // we're requiring SSE2 for both. 14083 if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) { 14084 SDValue N0 = N->getOperand(0); 14085 SDValue N1 = N->getOperand(1); 14086 SDValue CMP0 = N0->getOperand(1); 14087 SDValue CMP1 = N1->getOperand(1); 14088 DebugLoc DL = N->getDebugLoc(); 14089 14090 // The SETCCs should both refer to the same CMP. 14091 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1) 14092 return SDValue(); 14093 14094 SDValue CMP00 = CMP0->getOperand(0); 14095 SDValue CMP01 = CMP0->getOperand(1); 14096 EVT VT = CMP00.getValueType(); 14097 14098 if (VT == MVT::f32 || VT == MVT::f64) { 14099 bool ExpectingFlags = false; 14100 // Check for any users that want flags: 14101 for (SDNode::use_iterator UI = N->use_begin(), 14102 UE = N->use_end(); 14103 !ExpectingFlags && UI != UE; ++UI) 14104 switch (UI->getOpcode()) { 14105 default: 14106 case ISD::BR_CC: 14107 case ISD::BRCOND: 14108 case ISD::SELECT: 14109 ExpectingFlags = true; 14110 break; 14111 case ISD::CopyToReg: 14112 case ISD::SIGN_EXTEND: 14113 case ISD::ZERO_EXTEND: 14114 case ISD::ANY_EXTEND: 14115 break; 14116 } 14117 14118 if (!ExpectingFlags) { 14119 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0); 14120 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0); 14121 14122 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) { 14123 X86::CondCode tmp = cc0; 14124 cc0 = cc1; 14125 cc1 = tmp; 14126 } 14127 14128 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) || 14129 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) { 14130 bool is64BitFP = (CMP00.getValueType() == MVT::f64); 14131 X86ISD::NodeType NTOperator = is64BitFP ? 14132 X86ISD::FSETCCsd : X86ISD::FSETCCss; 14133 // FIXME: need symbolic constants for these magic numbers. 14134 // See X86ATTInstPrinter.cpp:printSSECC(). 14135 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4; 14136 SDValue OnesOrZeroesF = DAG.getNode(NTOperator, DL, MVT::f32, CMP00, CMP01, 14137 DAG.getConstant(x86cc, MVT::i8)); 14138 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, MVT::i32, 14139 OnesOrZeroesF); 14140 SDValue ANDed = DAG.getNode(ISD::AND, DL, MVT::i32, OnesOrZeroesI, 14141 DAG.getConstant(1, MVT::i32)); 14142 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); 14143 return OneBitOfTruth; 14144 } 14145 } 14146 } 14147 } 14148 return SDValue(); 14149} 14150 14151/// CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector 14152/// so it can be folded inside ANDNP. 14153static bool CanFoldXORWithAllOnes(const SDNode *N) { 14154 EVT VT = N->getValueType(0); 14155 14156 // Match direct AllOnes for 128 and 256-bit vectors 14157 if (ISD::isBuildVectorAllOnes(N)) 14158 return true; 14159 14160 // Look through a bit convert. 14161 if (N->getOpcode() == ISD::BITCAST) 14162 N = N->getOperand(0).getNode(); 14163 14164 // Sometimes the operand may come from a insert_subvector building a 256-bit 14165 // allones vector 14166 if (VT.getSizeInBits() == 256 && 14167 N->getOpcode() == ISD::INSERT_SUBVECTOR) { 14168 SDValue V1 = N->getOperand(0); 14169 SDValue V2 = N->getOperand(1); 14170 14171 if (V1.getOpcode() == ISD::INSERT_SUBVECTOR && 14172 V1.getOperand(0).getOpcode() == ISD::UNDEF && 14173 ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) && 14174 ISD::isBuildVectorAllOnes(V2.getNode())) 14175 return true; 14176 } 14177 14178 return false; 14179} 14180 14181static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, 14182 TargetLowering::DAGCombinerInfo &DCI, 14183 const X86Subtarget *Subtarget) { 14184 if (DCI.isBeforeLegalizeOps()) 14185 return SDValue(); 14186 14187 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); 14188 if (R.getNode()) 14189 return R; 14190 14191 EVT VT = N->getValueType(0); 14192 14193 // Create ANDN, BLSI, and BLSR instructions 14194 // BLSI is X & (-X) 14195 // BLSR is X & (X-1) 14196 if (Subtarget->hasBMI() && (VT == MVT::i32 || VT == MVT::i64)) { 14197 SDValue N0 = N->getOperand(0); 14198 SDValue N1 = N->getOperand(1); 14199 DebugLoc DL = N->getDebugLoc(); 14200 14201 // Check LHS for not 14202 if (N0.getOpcode() == ISD::XOR && isAllOnes(N0.getOperand(1))) 14203 return DAG.getNode(X86ISD::ANDN, DL, VT, N0.getOperand(0), N1); 14204 // Check RHS for not 14205 if (N1.getOpcode() == ISD::XOR && isAllOnes(N1.getOperand(1))) 14206 return DAG.getNode(X86ISD::ANDN, DL, VT, N1.getOperand(0), N0); 14207 14208 // Check LHS for neg 14209 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1) == N1 && 14210 isZero(N0.getOperand(0))) 14211 return DAG.getNode(X86ISD::BLSI, DL, VT, N1); 14212 14213 // Check RHS for neg 14214 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1) == N0 && 14215 isZero(N1.getOperand(0))) 14216 return DAG.getNode(X86ISD::BLSI, DL, VT, N0); 14217 14218 // Check LHS for X-1 14219 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1 && 14220 isAllOnes(N0.getOperand(1))) 14221 return DAG.getNode(X86ISD::BLSR, DL, VT, N1); 14222 14223 // Check RHS for X-1 14224 if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N0 && 14225 isAllOnes(N1.getOperand(1))) 14226 return DAG.getNode(X86ISD::BLSR, DL, VT, N0); 14227 14228 return SDValue(); 14229 } 14230 14231 // Want to form ANDNP nodes: 14232 // 1) In the hopes of then easily combining them with OR and AND nodes 14233 // to form PBLEND/PSIGN. 14234 // 2) To match ANDN packed intrinsics 14235 if (VT != MVT::v2i64 && VT != MVT::v4i64) 14236 return SDValue(); 14237 14238 SDValue N0 = N->getOperand(0); 14239 SDValue N1 = N->getOperand(1); 14240 DebugLoc DL = N->getDebugLoc(); 14241 14242 // Check LHS for vnot 14243 if (N0.getOpcode() == ISD::XOR && 14244 //ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode())) 14245 CanFoldXORWithAllOnes(N0.getOperand(1).getNode())) 14246 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1); 14247 14248 // Check RHS for vnot 14249 if (N1.getOpcode() == ISD::XOR && 14250 //ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode())) 14251 CanFoldXORWithAllOnes(N1.getOperand(1).getNode())) 14252 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0); 14253 14254 return SDValue(); 14255} 14256 14257static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, 14258 TargetLowering::DAGCombinerInfo &DCI, 14259 const X86Subtarget *Subtarget) { 14260 if (DCI.isBeforeLegalizeOps()) 14261 return SDValue(); 14262 14263 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); 14264 if (R.getNode()) 14265 return R; 14266 14267 EVT VT = N->getValueType(0); 14268 14269 SDValue N0 = N->getOperand(0); 14270 SDValue N1 = N->getOperand(1); 14271 14272 // look for psign/blend 14273 if (VT == MVT::v2i64 || VT == MVT::v4i64) { 14274 if (!Subtarget->hasSSSE3() || 14275 (VT == MVT::v4i64 && !Subtarget->hasAVX2())) 14276 return SDValue(); 14277 14278 // Canonicalize pandn to RHS 14279 if (N0.getOpcode() == X86ISD::ANDNP) 14280 std::swap(N0, N1); 14281 // or (and (m, y), (pandn m, x)) 14282 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) { 14283 SDValue Mask = N1.getOperand(0); 14284 SDValue X = N1.getOperand(1); 14285 SDValue Y; 14286 if (N0.getOperand(0) == Mask) 14287 Y = N0.getOperand(1); 14288 if (N0.getOperand(1) == Mask) 14289 Y = N0.getOperand(0); 14290 14291 // Check to see if the mask appeared in both the AND and ANDNP and 14292 if (!Y.getNode()) 14293 return SDValue(); 14294 14295 // Validate that X, Y, and Mask are BIT_CONVERTS, and see through them. 14296 // Look through mask bitcast. 14297 if (Mask.getOpcode() == ISD::BITCAST) 14298 Mask = Mask.getOperand(0); 14299 if (X.getOpcode() == ISD::BITCAST) 14300 X = X.getOperand(0); 14301 if (Y.getOpcode() == ISD::BITCAST) 14302 Y = Y.getOperand(0); 14303 14304 EVT MaskVT = Mask.getValueType(); 14305 14306 // Validate that the Mask operand is a vector sra node. 14307 // FIXME: what to do for bytes, since there is a psignb/pblendvb, but 14308 // there is no psrai.b 14309 if (Mask.getOpcode() != X86ISD::VSRAI) 14310 return SDValue(); 14311 14312 // Check that the SRA is all signbits. 14313 SDValue SraC = Mask.getOperand(1); 14314 unsigned SraAmt = cast<ConstantSDNode>(SraC)->getZExtValue(); 14315 unsigned EltBits = MaskVT.getVectorElementType().getSizeInBits(); 14316 if ((SraAmt + 1) != EltBits) 14317 return SDValue(); 14318 14319 DebugLoc DL = N->getDebugLoc(); 14320 14321 // Now we know we at least have a plendvb with the mask val. See if 14322 // we can form a psignb/w/d. 14323 // psign = x.type == y.type == mask.type && y = sub(0, x); 14324 if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X && 14325 ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) && 14326 X.getValueType() == MaskVT && Y.getValueType() == MaskVT) { 14327 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) && 14328 "Unsupported VT for PSIGN"); 14329 Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0)); 14330 return DAG.getNode(ISD::BITCAST, DL, VT, Mask); 14331 } 14332 // PBLENDVB only available on SSE 4.1 14333 if (!Subtarget->hasSSE41()) 14334 return SDValue(); 14335 14336 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8; 14337 14338 X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X); 14339 Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y); 14340 Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask); 14341 Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X); 14342 return DAG.getNode(ISD::BITCAST, DL, VT, Mask); 14343 } 14344 } 14345 14346 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) 14347 return SDValue(); 14348 14349 // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c) 14350 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) 14351 std::swap(N0, N1); 14352 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) 14353 return SDValue(); 14354 if (!N0.hasOneUse() || !N1.hasOneUse()) 14355 return SDValue(); 14356 14357 SDValue ShAmt0 = N0.getOperand(1); 14358 if (ShAmt0.getValueType() != MVT::i8) 14359 return SDValue(); 14360 SDValue ShAmt1 = N1.getOperand(1); 14361 if (ShAmt1.getValueType() != MVT::i8) 14362 return SDValue(); 14363 if (ShAmt0.getOpcode() == ISD::TRUNCATE) 14364 ShAmt0 = ShAmt0.getOperand(0); 14365 if (ShAmt1.getOpcode() == ISD::TRUNCATE) 14366 ShAmt1 = ShAmt1.getOperand(0); 14367 14368 DebugLoc DL = N->getDebugLoc(); 14369 unsigned Opc = X86ISD::SHLD; 14370 SDValue Op0 = N0.getOperand(0); 14371 SDValue Op1 = N1.getOperand(0); 14372 if (ShAmt0.getOpcode() == ISD::SUB) { 14373 Opc = X86ISD::SHRD; 14374 std::swap(Op0, Op1); 14375 std::swap(ShAmt0, ShAmt1); 14376 } 14377 14378 unsigned Bits = VT.getSizeInBits(); 14379 if (ShAmt1.getOpcode() == ISD::SUB) { 14380 SDValue Sum = ShAmt1.getOperand(0); 14381 if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) { 14382 SDValue ShAmt1Op1 = ShAmt1.getOperand(1); 14383 if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE) 14384 ShAmt1Op1 = ShAmt1Op1.getOperand(0); 14385 if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0) 14386 return DAG.getNode(Opc, DL, VT, 14387 Op0, Op1, 14388 DAG.getNode(ISD::TRUNCATE, DL, 14389 MVT::i8, ShAmt0)); 14390 } 14391 } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) { 14392 ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0); 14393 if (ShAmt0C && 14394 ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits) 14395 return DAG.getNode(Opc, DL, VT, 14396 N0.getOperand(0), N1.getOperand(0), 14397 DAG.getNode(ISD::TRUNCATE, DL, 14398 MVT::i8, ShAmt0)); 14399 } 14400 14401 return SDValue(); 14402} 14403 14404// PerformXorCombine - Attempts to turn XOR nodes into BLSMSK nodes 14405static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG, 14406 TargetLowering::DAGCombinerInfo &DCI, 14407 const X86Subtarget *Subtarget) { 14408 if (DCI.isBeforeLegalizeOps()) 14409 return SDValue(); 14410 14411 EVT VT = N->getValueType(0); 14412 14413 if (VT != MVT::i32 && VT != MVT::i64) 14414 return SDValue(); 14415 14416 assert(Subtarget->hasBMI() && "Creating BLSMSK requires BMI instructions"); 14417 14418 // Create BLSMSK instructions by finding X ^ (X-1) 14419 SDValue N0 = N->getOperand(0); 14420 SDValue N1 = N->getOperand(1); 14421 DebugLoc DL = N->getDebugLoc(); 14422 14423 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1 && 14424 isAllOnes(N0.getOperand(1))) 14425 return DAG.getNode(X86ISD::BLSMSK, DL, VT, N1); 14426 14427 if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N0 && 14428 isAllOnes(N1.getOperand(1))) 14429 return DAG.getNode(X86ISD::BLSMSK, DL, VT, N0); 14430 14431 return SDValue(); 14432} 14433 14434/// PerformLOADCombine - Do target-specific dag combines on LOAD nodes. 14435static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG, 14436 const X86Subtarget *Subtarget) { 14437 LoadSDNode *Ld = cast<LoadSDNode>(N); 14438 EVT RegVT = Ld->getValueType(0); 14439 EVT MemVT = Ld->getMemoryVT(); 14440 DebugLoc dl = Ld->getDebugLoc(); 14441 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 14442 14443 ISD::LoadExtType Ext = Ld->getExtensionType(); 14444 14445 // If this is a vector EXT Load then attempt to optimize it using a 14446 // shuffle. We need SSE4 for the shuffles. 14447 // TODO: It is possible to support ZExt by zeroing the undef values 14448 // during the shuffle phase or after the shuffle. 14449 if (RegVT.isVector() && RegVT.isInteger() && 14450 Ext == ISD::EXTLOAD && Subtarget->hasSSE41()) { 14451 assert(MemVT != RegVT && "Cannot extend to the same type"); 14452 assert(MemVT.isVector() && "Must load a vector from memory"); 14453 14454 unsigned NumElems = RegVT.getVectorNumElements(); 14455 unsigned RegSz = RegVT.getSizeInBits(); 14456 unsigned MemSz = MemVT.getSizeInBits(); 14457 assert(RegSz > MemSz && "Register size must be greater than the mem size"); 14458 // All sizes must be a power of two 14459 if (!isPowerOf2_32(RegSz * MemSz * NumElems)) return SDValue(); 14460 14461 // Attempt to load the original value using a single load op. 14462 // Find a scalar type which is equal to the loaded word size. 14463 MVT SclrLoadTy = MVT::i8; 14464 for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE; 14465 tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) { 14466 MVT Tp = (MVT::SimpleValueType)tp; 14467 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() == MemSz) { 14468 SclrLoadTy = Tp; 14469 break; 14470 } 14471 } 14472 14473 // Proceed if a load word is found. 14474 if (SclrLoadTy.getSizeInBits() != MemSz) return SDValue(); 14475 14476 EVT LoadUnitVecVT = EVT::getVectorVT(*DAG.getContext(), SclrLoadTy, 14477 RegSz/SclrLoadTy.getSizeInBits()); 14478 14479 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), 14480 RegSz/MemVT.getScalarType().getSizeInBits()); 14481 // Can't shuffle using an illegal type. 14482 if (!TLI.isTypeLegal(WideVecVT)) return SDValue(); 14483 14484 // Perform a single load. 14485 SDValue ScalarLoad = DAG.getLoad(SclrLoadTy, dl, Ld->getChain(), 14486 Ld->getBasePtr(), 14487 Ld->getPointerInfo(), Ld->isVolatile(), 14488 Ld->isNonTemporal(), Ld->isInvariant(), 14489 Ld->getAlignment()); 14490 14491 // Insert the word loaded into a vector. 14492 SDValue ScalarInVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 14493 LoadUnitVecVT, ScalarLoad); 14494 14495 // Bitcast the loaded value to a vector of the original element type, in 14496 // the size of the target vector type. 14497 SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, 14498 ScalarInVector); 14499 unsigned SizeRatio = RegSz/MemSz; 14500 14501 // Redistribute the loaded elements into the different locations. 14502 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1); 14503 for (unsigned i = 0; i != NumElems; ++i) 14504 ShuffleVec[i*SizeRatio] = i; 14505 14506 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec, 14507 DAG.getUNDEF(WideVecVT), 14508 &ShuffleVec[0]); 14509 14510 // Bitcast to the requested type. 14511 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff); 14512 // Replace the original load with the new sequence 14513 // and return the new chain. 14514 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Shuff); 14515 return SDValue(ScalarLoad.getNode(), 1); 14516 } 14517 14518 return SDValue(); 14519} 14520 14521/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 14522static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 14523 const X86Subtarget *Subtarget) { 14524 StoreSDNode *St = cast<StoreSDNode>(N); 14525 EVT VT = St->getValue().getValueType(); 14526 EVT StVT = St->getMemoryVT(); 14527 DebugLoc dl = St->getDebugLoc(); 14528 SDValue StoredVal = St->getOperand(1); 14529 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 14530 14531 // If we are saving a concatenation of two XMM registers, perform two stores. 14532 // This is better in Sandy Bridge cause one 256-bit mem op is done via two 14533 // 128-bit ones. If in the future the cost becomes only one memory access the 14534 // first version would be better. 14535 if (VT.getSizeInBits() == 256 && 14536 StoredVal.getNode()->getOpcode() == ISD::CONCAT_VECTORS && 14537 StoredVal.getNumOperands() == 2) { 14538 14539 SDValue Value0 = StoredVal.getOperand(0); 14540 SDValue Value1 = StoredVal.getOperand(1); 14541 14542 SDValue Stride = DAG.getConstant(16, TLI.getPointerTy()); 14543 SDValue Ptr0 = St->getBasePtr(); 14544 SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride); 14545 14546 SDValue Ch0 = DAG.getStore(St->getChain(), dl, Value0, Ptr0, 14547 St->getPointerInfo(), St->isVolatile(), 14548 St->isNonTemporal(), St->getAlignment()); 14549 SDValue Ch1 = DAG.getStore(St->getChain(), dl, Value1, Ptr1, 14550 St->getPointerInfo(), St->isVolatile(), 14551 St->isNonTemporal(), St->getAlignment()); 14552 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); 14553 } 14554 14555 // Optimize trunc store (of multiple scalars) to shuffle and store. 14556 // First, pack all of the elements in one place. Next, store to memory 14557 // in fewer chunks. 14558 if (St->isTruncatingStore() && VT.isVector()) { 14559 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 14560 unsigned NumElems = VT.getVectorNumElements(); 14561 assert(StVT != VT && "Cannot truncate to the same type"); 14562 unsigned FromSz = VT.getVectorElementType().getSizeInBits(); 14563 unsigned ToSz = StVT.getVectorElementType().getSizeInBits(); 14564 14565 // From, To sizes and ElemCount must be pow of two 14566 if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue(); 14567 // We are going to use the original vector elt for storing. 14568 // Accumulated smaller vector elements must be a multiple of the store size. 14569 if (0 != (NumElems * FromSz) % ToSz) return SDValue(); 14570 14571 unsigned SizeRatio = FromSz / ToSz; 14572 14573 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits()); 14574 14575 // Create a type on which we perform the shuffle 14576 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), 14577 StVT.getScalarType(), NumElems*SizeRatio); 14578 14579 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits()); 14580 14581 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue()); 14582 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1); 14583 for (unsigned i = 0; i != NumElems; ++i) 14584 ShuffleVec[i] = i * SizeRatio; 14585 14586 // Can't shuffle using an illegal type 14587 if (!TLI.isTypeLegal(WideVecVT)) return SDValue(); 14588 14589 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec, 14590 DAG.getUNDEF(WideVecVT), 14591 &ShuffleVec[0]); 14592 // At this point all of the data is stored at the bottom of the 14593 // register. We now need to save it to mem. 14594 14595 // Find the largest store unit 14596 MVT StoreType = MVT::i8; 14597 for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE; 14598 tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) { 14599 MVT Tp = (MVT::SimpleValueType)tp; 14600 if (TLI.isTypeLegal(Tp) && StoreType.getSizeInBits() < NumElems * ToSz) 14601 StoreType = Tp; 14602 } 14603 14604 // Bitcast the original vector into a vector of store-size units 14605 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(), 14606 StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits()); 14607 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits()); 14608 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff); 14609 SmallVector<SDValue, 8> Chains; 14610 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8, 14611 TLI.getPointerTy()); 14612 SDValue Ptr = St->getBasePtr(); 14613 14614 // Perform one or more big stores into memory. 14615 for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) { 14616 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 14617 StoreType, ShuffWide, 14618 DAG.getIntPtrConstant(i)); 14619 SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr, 14620 St->getPointerInfo(), St->isVolatile(), 14621 St->isNonTemporal(), St->getAlignment()); 14622 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 14623 Chains.push_back(Ch); 14624 } 14625 14626 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], 14627 Chains.size()); 14628 } 14629 14630 14631 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 14632 // the FP state in cases where an emms may be missing. 14633 // A preferable solution to the general problem is to figure out the right 14634 // places to insert EMMS. This qualifies as a quick hack. 14635 14636 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode. 14637 if (VT.getSizeInBits() != 64) 14638 return SDValue(); 14639 14640 const Function *F = DAG.getMachineFunction().getFunction(); 14641 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat); 14642 bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps 14643 && Subtarget->hasSSE2(); 14644 if ((VT.isVector() || 14645 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && 14646 isa<LoadSDNode>(St->getValue()) && 14647 !cast<LoadSDNode>(St->getValue())->isVolatile() && 14648 St->getChain().hasOneUse() && !St->isVolatile()) { 14649 SDNode* LdVal = St->getValue().getNode(); 14650 LoadSDNode *Ld = 0; 14651 int TokenFactorIndex = -1; 14652 SmallVector<SDValue, 8> Ops; 14653 SDNode* ChainVal = St->getChain().getNode(); 14654 // Must be a store of a load. We currently handle two cases: the load 14655 // is a direct child, and it's under an intervening TokenFactor. It is 14656 // possible to dig deeper under nested TokenFactors. 14657 if (ChainVal == LdVal) 14658 Ld = cast<LoadSDNode>(St->getChain()); 14659 else if (St->getValue().hasOneUse() && 14660 ChainVal->getOpcode() == ISD::TokenFactor) { 14661 for (unsigned i = 0, e = ChainVal->getNumOperands(); i != e; ++i) { 14662 if (ChainVal->getOperand(i).getNode() == LdVal) { 14663 TokenFactorIndex = i; 14664 Ld = cast<LoadSDNode>(St->getValue()); 14665 } else 14666 Ops.push_back(ChainVal->getOperand(i)); 14667 } 14668 } 14669 14670 if (!Ld || !ISD::isNormalLoad(Ld)) 14671 return SDValue(); 14672 14673 // If this is not the MMX case, i.e. we are just turning i64 load/store 14674 // into f64 load/store, avoid the transformation if there are multiple 14675 // uses of the loaded value. 14676 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0)) 14677 return SDValue(); 14678 14679 DebugLoc LdDL = Ld->getDebugLoc(); 14680 DebugLoc StDL = N->getDebugLoc(); 14681 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 14682 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store 14683 // pair instead. 14684 if (Subtarget->is64Bit() || F64IsLegal) { 14685 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; 14686 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(), 14687 Ld->getPointerInfo(), Ld->isVolatile(), 14688 Ld->isNonTemporal(), Ld->isInvariant(), 14689 Ld->getAlignment()); 14690 SDValue NewChain = NewLd.getValue(1); 14691 if (TokenFactorIndex != -1) { 14692 Ops.push_back(NewChain); 14693 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 14694 Ops.size()); 14695 } 14696 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(), 14697 St->getPointerInfo(), 14698 St->isVolatile(), St->isNonTemporal(), 14699 St->getAlignment()); 14700 } 14701 14702 // Otherwise, lower to two pairs of 32-bit loads / stores. 14703 SDValue LoAddr = Ld->getBasePtr(); 14704 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, 14705 DAG.getConstant(4, MVT::i32)); 14706 14707 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, 14708 Ld->getPointerInfo(), 14709 Ld->isVolatile(), Ld->isNonTemporal(), 14710 Ld->isInvariant(), Ld->getAlignment()); 14711 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, 14712 Ld->getPointerInfo().getWithOffset(4), 14713 Ld->isVolatile(), Ld->isNonTemporal(), 14714 Ld->isInvariant(), 14715 MinAlign(Ld->getAlignment(), 4)); 14716 14717 SDValue NewChain = LoLd.getValue(1); 14718 if (TokenFactorIndex != -1) { 14719 Ops.push_back(LoLd); 14720 Ops.push_back(HiLd); 14721 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 14722 Ops.size()); 14723 } 14724 14725 LoAddr = St->getBasePtr(); 14726 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, 14727 DAG.getConstant(4, MVT::i32)); 14728 14729 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr, 14730 St->getPointerInfo(), 14731 St->isVolatile(), St->isNonTemporal(), 14732 St->getAlignment()); 14733 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr, 14734 St->getPointerInfo().getWithOffset(4), 14735 St->isVolatile(), 14736 St->isNonTemporal(), 14737 MinAlign(St->getAlignment(), 4)); 14738 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); 14739 } 14740 return SDValue(); 14741} 14742 14743/// isHorizontalBinOp - Return 'true' if this vector operation is "horizontal" 14744/// and return the operands for the horizontal operation in LHS and RHS. A 14745/// horizontal operation performs the binary operation on successive elements 14746/// of its first operand, then on successive elements of its second operand, 14747/// returning the resulting values in a vector. For example, if 14748/// A = < float a0, float a1, float a2, float a3 > 14749/// and 14750/// B = < float b0, float b1, float b2, float b3 > 14751/// then the result of doing a horizontal operation on A and B is 14752/// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. 14753/// In short, LHS and RHS are inspected to see if LHS op RHS is of the form 14754/// A horizontal-op B, for some already available A and B, and if so then LHS is 14755/// set to A, RHS to B, and the routine returns 'true'. 14756/// Note that the binary operation should have the property that if one of the 14757/// operands is UNDEF then the result is UNDEF. 14758static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, bool IsCommutative) { 14759 // Look for the following pattern: if 14760 // A = < float a0, float a1, float a2, float a3 > 14761 // B = < float b0, float b1, float b2, float b3 > 14762 // and 14763 // LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6> 14764 // RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7> 14765 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 > 14766 // which is A horizontal-op B. 14767 14768 // At least one of the operands should be a vector shuffle. 14769 if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE && 14770 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) 14771 return false; 14772 14773 EVT VT = LHS.getValueType(); 14774 14775 assert((VT.is128BitVector() || VT.is256BitVector()) && 14776 "Unsupported vector type for horizontal add/sub"); 14777 14778 // Handle 128 and 256-bit vector lengths. AVX defines horizontal add/sub to 14779 // operate independently on 128-bit lanes. 14780 unsigned NumElts = VT.getVectorNumElements(); 14781 unsigned NumLanes = VT.getSizeInBits()/128; 14782 unsigned NumLaneElts = NumElts / NumLanes; 14783 assert((NumLaneElts % 2 == 0) && 14784 "Vector type should have an even number of elements in each lane"); 14785 unsigned HalfLaneElts = NumLaneElts/2; 14786 14787 // View LHS in the form 14788 // LHS = VECTOR_SHUFFLE A, B, LMask 14789 // If LHS is not a shuffle then pretend it is the shuffle 14790 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1> 14791 // NOTE: in what follows a default initialized SDValue represents an UNDEF of 14792 // type VT. 14793 SDValue A, B; 14794 SmallVector<int, 16> LMask(NumElts); 14795 if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) { 14796 if (LHS.getOperand(0).getOpcode() != ISD::UNDEF) 14797 A = LHS.getOperand(0); 14798 if (LHS.getOperand(1).getOpcode() != ISD::UNDEF) 14799 B = LHS.getOperand(1); 14800 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(LHS.getNode())->getMask(); 14801 std::copy(Mask.begin(), Mask.end(), LMask.begin()); 14802 } else { 14803 if (LHS.getOpcode() != ISD::UNDEF) 14804 A = LHS; 14805 for (unsigned i = 0; i != NumElts; ++i) 14806 LMask[i] = i; 14807 } 14808 14809 // Likewise, view RHS in the form 14810 // RHS = VECTOR_SHUFFLE C, D, RMask 14811 SDValue C, D; 14812 SmallVector<int, 16> RMask(NumElts); 14813 if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) { 14814 if (RHS.getOperand(0).getOpcode() != ISD::UNDEF) 14815 C = RHS.getOperand(0); 14816 if (RHS.getOperand(1).getOpcode() != ISD::UNDEF) 14817 D = RHS.getOperand(1); 14818 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(RHS.getNode())->getMask(); 14819 std::copy(Mask.begin(), Mask.end(), RMask.begin()); 14820 } else { 14821 if (RHS.getOpcode() != ISD::UNDEF) 14822 C = RHS; 14823 for (unsigned i = 0; i != NumElts; ++i) 14824 RMask[i] = i; 14825 } 14826 14827 // Check that the shuffles are both shuffling the same vectors. 14828 if (!(A == C && B == D) && !(A == D && B == C)) 14829 return false; 14830 14831 // If everything is UNDEF then bail out: it would be better to fold to UNDEF. 14832 if (!A.getNode() && !B.getNode()) 14833 return false; 14834 14835 // If A and B occur in reverse order in RHS, then "swap" them (which means 14836 // rewriting the mask). 14837 if (A != C) 14838 CommuteVectorShuffleMask(RMask, NumElts); 14839 14840 // At this point LHS and RHS are equivalent to 14841 // LHS = VECTOR_SHUFFLE A, B, LMask 14842 // RHS = VECTOR_SHUFFLE A, B, RMask 14843 // Check that the masks correspond to performing a horizontal operation. 14844 for (unsigned i = 0; i != NumElts; ++i) { 14845 int LIdx = LMask[i], RIdx = RMask[i]; 14846 14847 // Ignore any UNDEF components. 14848 if (LIdx < 0 || RIdx < 0 || 14849 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) || 14850 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts))) 14851 continue; 14852 14853 // Check that successive elements are being operated on. If not, this is 14854 // not a horizontal operation. 14855 unsigned Src = (i/HalfLaneElts) % 2; // each lane is split between srcs 14856 unsigned LaneStart = (i/NumLaneElts) * NumLaneElts; 14857 int Index = 2*(i%HalfLaneElts) + NumElts*Src + LaneStart; 14858 if (!(LIdx == Index && RIdx == Index + 1) && 14859 !(IsCommutative && LIdx == Index + 1 && RIdx == Index)) 14860 return false; 14861 } 14862 14863 LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it. 14864 RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it. 14865 return true; 14866} 14867 14868/// PerformFADDCombine - Do target-specific dag combines on floating point adds. 14869static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, 14870 const X86Subtarget *Subtarget) { 14871 EVT VT = N->getValueType(0); 14872 SDValue LHS = N->getOperand(0); 14873 SDValue RHS = N->getOperand(1); 14874 14875 // Try to synthesize horizontal adds from adds of shuffles. 14876 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || 14877 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && 14878 isHorizontalBinOp(LHS, RHS, true)) 14879 return DAG.getNode(X86ISD::FHADD, N->getDebugLoc(), VT, LHS, RHS); 14880 return SDValue(); 14881} 14882 14883/// PerformFSUBCombine - Do target-specific dag combines on floating point subs. 14884static SDValue PerformFSUBCombine(SDNode *N, SelectionDAG &DAG, 14885 const X86Subtarget *Subtarget) { 14886 EVT VT = N->getValueType(0); 14887 SDValue LHS = N->getOperand(0); 14888 SDValue RHS = N->getOperand(1); 14889 14890 // Try to synthesize horizontal subs from subs of shuffles. 14891 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || 14892 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && 14893 isHorizontalBinOp(LHS, RHS, false)) 14894 return DAG.getNode(X86ISD::FHSUB, N->getDebugLoc(), VT, LHS, RHS); 14895 return SDValue(); 14896} 14897 14898/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 14899/// X86ISD::FXOR nodes. 14900static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 14901 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 14902 // F[X]OR(0.0, x) -> x 14903 // F[X]OR(x, 0.0) -> x 14904 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 14905 if (C->getValueAPF().isPosZero()) 14906 return N->getOperand(1); 14907 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 14908 if (C->getValueAPF().isPosZero()) 14909 return N->getOperand(0); 14910 return SDValue(); 14911} 14912 14913/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 14914static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 14915 // FAND(0.0, x) -> 0.0 14916 // FAND(x, 0.0) -> 0.0 14917 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 14918 if (C->getValueAPF().isPosZero()) 14919 return N->getOperand(0); 14920 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 14921 if (C->getValueAPF().isPosZero()) 14922 return N->getOperand(1); 14923 return SDValue(); 14924} 14925 14926static SDValue PerformBTCombine(SDNode *N, 14927 SelectionDAG &DAG, 14928 TargetLowering::DAGCombinerInfo &DCI) { 14929 // BT ignores high bits in the bit index operand. 14930 SDValue Op1 = N->getOperand(1); 14931 if (Op1.hasOneUse()) { 14932 unsigned BitWidth = Op1.getValueSizeInBits(); 14933 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth)); 14934 APInt KnownZero, KnownOne; 14935 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), 14936 !DCI.isBeforeLegalizeOps()); 14937 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 14938 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) || 14939 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO)) 14940 DCI.CommitTargetLoweringOpt(TLO); 14941 } 14942 return SDValue(); 14943} 14944 14945static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) { 14946 SDValue Op = N->getOperand(0); 14947 if (Op.getOpcode() == ISD::BITCAST) 14948 Op = Op.getOperand(0); 14949 EVT VT = N->getValueType(0), OpVT = Op.getValueType(); 14950 if (Op.getOpcode() == X86ISD::VZEXT_LOAD && 14951 VT.getVectorElementType().getSizeInBits() == 14952 OpVT.getVectorElementType().getSizeInBits()) { 14953 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, Op); 14954 } 14955 return SDValue(); 14956} 14957 14958static SDValue PerformSExtCombine(SDNode *N, SelectionDAG &DAG, 14959 TargetLowering::DAGCombinerInfo &DCI, 14960 const X86Subtarget *Subtarget) { 14961 if (!DCI.isBeforeLegalizeOps()) 14962 return SDValue(); 14963 14964 if (!Subtarget->hasAVX()) 14965 return SDValue(); 14966 14967 EVT VT = N->getValueType(0); 14968 SDValue Op = N->getOperand(0); 14969 EVT OpVT = Op.getValueType(); 14970 DebugLoc dl = N->getDebugLoc(); 14971 14972 if ((VT == MVT::v4i64 && OpVT == MVT::v4i32) || 14973 (VT == MVT::v8i32 && OpVT == MVT::v8i16)) { 14974 14975 if (Subtarget->hasAVX2()) 14976 return DAG.getNode(X86ISD::VSEXT_MOVL, dl, VT, Op); 14977 14978 // Optimize vectors in AVX mode 14979 // Sign extend v8i16 to v8i32 and 14980 // v4i32 to v4i64 14981 // 14982 // Divide input vector into two parts 14983 // for v4i32 the shuffle mask will be { 0, 1, -1, -1} {2, 3, -1, -1} 14984 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32 14985 // concat the vectors to original VT 14986 14987 unsigned NumElems = OpVT.getVectorNumElements(); 14988 SmallVector<int,8> ShufMask1(NumElems, -1); 14989 for (unsigned i = 0; i != NumElems/2; ++i) 14990 ShufMask1[i] = i; 14991 14992 SDValue OpLo = DAG.getVectorShuffle(OpVT, dl, Op, DAG.getUNDEF(OpVT), 14993 &ShufMask1[0]); 14994 14995 SmallVector<int,8> ShufMask2(NumElems, -1); 14996 for (unsigned i = 0; i != NumElems/2; ++i) 14997 ShufMask2[i] = i + NumElems/2; 14998 14999 SDValue OpHi = DAG.getVectorShuffle(OpVT, dl, Op, DAG.getUNDEF(OpVT), 15000 &ShufMask2[0]); 15001 15002 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), 15003 VT.getVectorNumElements()/2); 15004 15005 OpLo = DAG.getNode(X86ISD::VSEXT_MOVL, dl, HalfVT, OpLo); 15006 OpHi = DAG.getNode(X86ISD::VSEXT_MOVL, dl, HalfVT, OpHi); 15007 15008 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); 15009 } 15010 return SDValue(); 15011} 15012 15013static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG, 15014 TargetLowering::DAGCombinerInfo &DCI, 15015 const X86Subtarget *Subtarget) { 15016 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) -> 15017 // (and (i32 x86isd::setcc_carry), 1) 15018 // This eliminates the zext. This transformation is necessary because 15019 // ISD::SETCC is always legalized to i8. 15020 DebugLoc dl = N->getDebugLoc(); 15021 SDValue N0 = N->getOperand(0); 15022 EVT VT = N->getValueType(0); 15023 EVT OpVT = N0.getValueType(); 15024 15025 if (N0.getOpcode() == ISD::AND && 15026 N0.hasOneUse() && 15027 N0.getOperand(0).hasOneUse()) { 15028 SDValue N00 = N0.getOperand(0); 15029 if (N00.getOpcode() != X86ISD::SETCC_CARRY) 15030 return SDValue(); 15031 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 15032 if (!C || C->getZExtValue() != 1) 15033 return SDValue(); 15034 return DAG.getNode(ISD::AND, dl, VT, 15035 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, 15036 N00.getOperand(0), N00.getOperand(1)), 15037 DAG.getConstant(1, VT)); 15038 } 15039 15040 // Optimize vectors in AVX mode: 15041 // 15042 // v8i16 -> v8i32 15043 // Use vpunpcklwd for 4 lower elements v8i16 -> v4i32. 15044 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32. 15045 // Concat upper and lower parts. 15046 // 15047 // v4i32 -> v4i64 15048 // Use vpunpckldq for 4 lower elements v4i32 -> v2i64. 15049 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64. 15050 // Concat upper and lower parts. 15051 // 15052 if (!DCI.isBeforeLegalizeOps()) 15053 return SDValue(); 15054 15055 if (!Subtarget->hasAVX()) 15056 return SDValue(); 15057 15058 if (((VT == MVT::v8i32) && (OpVT == MVT::v8i16)) || 15059 ((VT == MVT::v4i64) && (OpVT == MVT::v4i32))) { 15060 15061 if (Subtarget->hasAVX2()) 15062 return DAG.getNode(X86ISD::VZEXT_MOVL, dl, VT, N0); 15063 15064 SDValue ZeroVec = getZeroVector(OpVT, Subtarget, DAG, dl); 15065 SDValue OpLo = getUnpackl(DAG, dl, OpVT, N0, ZeroVec); 15066 SDValue OpHi = getUnpackh(DAG, dl, OpVT, N0, ZeroVec); 15067 15068 EVT HVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 15069 VT.getVectorNumElements()/2); 15070 15071 OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo); 15072 OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi); 15073 15074 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); 15075 } 15076 15077 return SDValue(); 15078} 15079 15080// Optimize x == -y --> x+y == 0 15081// x != -y --> x+y != 0 15082static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG) { 15083 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 15084 SDValue LHS = N->getOperand(0); 15085 SDValue RHS = N->getOperand(1); 15086 15087 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB) 15088 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0))) 15089 if (C->getAPIntValue() == 0 && LHS.hasOneUse()) { 15090 SDValue addV = DAG.getNode(ISD::ADD, N->getDebugLoc(), 15091 LHS.getValueType(), RHS, LHS.getOperand(1)); 15092 return DAG.getSetCC(N->getDebugLoc(), N->getValueType(0), 15093 addV, DAG.getConstant(0, addV.getValueType()), CC); 15094 } 15095 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB) 15096 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getOperand(0))) 15097 if (C->getAPIntValue() == 0 && RHS.hasOneUse()) { 15098 SDValue addV = DAG.getNode(ISD::ADD, N->getDebugLoc(), 15099 RHS.getValueType(), LHS, RHS.getOperand(1)); 15100 return DAG.getSetCC(N->getDebugLoc(), N->getValueType(0), 15101 addV, DAG.getConstant(0, addV.getValueType()), CC); 15102 } 15103 return SDValue(); 15104} 15105 15106// Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT 15107static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG) { 15108 unsigned X86CC = N->getConstantOperandVal(0); 15109 SDValue EFLAG = N->getOperand(1); 15110 DebugLoc DL = N->getDebugLoc(); 15111 15112 // Materialize "setb reg" as "sbb reg,reg", since it can be extended without 15113 // a zext and produces an all-ones bit which is more useful than 0/1 in some 15114 // cases. 15115 if (X86CC == X86::COND_B) 15116 return DAG.getNode(ISD::AND, DL, MVT::i8, 15117 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, 15118 DAG.getConstant(X86CC, MVT::i8), EFLAG), 15119 DAG.getConstant(1, MVT::i8)); 15120 15121 return SDValue(); 15122} 15123 15124static SDValue PerformUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG) { 15125 SDValue Op0 = N->getOperand(0); 15126 EVT InVT = Op0->getValueType(0); 15127 15128 // UINT_TO_FP(v4i8) -> SINT_TO_FP(ZEXT(v4i8 to v4i32)) 15129 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) { 15130 DebugLoc dl = N->getDebugLoc(); 15131 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32; 15132 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0); 15133 // Notice that we use SINT_TO_FP because we know that the high bits 15134 // are zero and SINT_TO_FP is better supported by the hardware. 15135 return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P); 15136 } 15137 15138 return SDValue(); 15139} 15140 15141static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, 15142 const X86TargetLowering *XTLI) { 15143 SDValue Op0 = N->getOperand(0); 15144 EVT InVT = Op0->getValueType(0); 15145 15146 // SINT_TO_FP(v4i8) -> SINT_TO_FP(SEXT(v4i8 to v4i32)) 15147 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) { 15148 DebugLoc dl = N->getDebugLoc(); 15149 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32; 15150 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); 15151 return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P); 15152 } 15153 15154 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have 15155 // a 32-bit target where SSE doesn't support i64->FP operations. 15156 if (Op0.getOpcode() == ISD::LOAD) { 15157 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode()); 15158 EVT VT = Ld->getValueType(0); 15159 if (!Ld->isVolatile() && !N->getValueType(0).isVector() && 15160 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() && 15161 !XTLI->getSubtarget()->is64Bit() && 15162 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) { 15163 SDValue FILDChain = XTLI->BuildFILD(SDValue(N, 0), Ld->getValueType(0), 15164 Ld->getChain(), Op0, DAG); 15165 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1)); 15166 return FILDChain; 15167 } 15168 } 15169 return SDValue(); 15170} 15171 15172static SDValue PerformFP_TO_SINTCombine(SDNode *N, SelectionDAG &DAG) { 15173 EVT VT = N->getValueType(0); 15174 15175 // v4i8 = FP_TO_SINT() -> v4i8 = TRUNCATE (V4i32 = FP_TO_SINT() 15176 if (VT == MVT::v8i8 || VT == MVT::v4i8) { 15177 DebugLoc dl = N->getDebugLoc(); 15178 MVT DstVT = VT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32; 15179 SDValue I = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, N->getOperand(0)); 15180 return DAG.getNode(ISD::TRUNCATE, dl, VT, I); 15181 } 15182 15183 return SDValue(); 15184} 15185 15186// Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS 15187static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG, 15188 X86TargetLowering::DAGCombinerInfo &DCI) { 15189 // If the LHS and RHS of the ADC node are zero, then it can't overflow and 15190 // the result is either zero or one (depending on the input carry bit). 15191 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1. 15192 if (X86::isZeroNode(N->getOperand(0)) && 15193 X86::isZeroNode(N->getOperand(1)) && 15194 // We don't have a good way to replace an EFLAGS use, so only do this when 15195 // dead right now. 15196 SDValue(N, 1).use_empty()) { 15197 DebugLoc DL = N->getDebugLoc(); 15198 EVT VT = N->getValueType(0); 15199 SDValue CarryOut = DAG.getConstant(0, N->getValueType(1)); 15200 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT, 15201 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, 15202 DAG.getConstant(X86::COND_B,MVT::i8), 15203 N->getOperand(2)), 15204 DAG.getConstant(1, VT)); 15205 return DCI.CombineTo(N, Res1, CarryOut); 15206 } 15207 15208 return SDValue(); 15209} 15210 15211// fold (add Y, (sete X, 0)) -> adc 0, Y 15212// (add Y, (setne X, 0)) -> sbb -1, Y 15213// (sub (sete X, 0), Y) -> sbb 0, Y 15214// (sub (setne X, 0), Y) -> adc -1, Y 15215static SDValue OptimizeConditionalInDecrement(SDNode *N, SelectionDAG &DAG) { 15216 DebugLoc DL = N->getDebugLoc(); 15217 15218 // Look through ZExts. 15219 SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0); 15220 if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse()) 15221 return SDValue(); 15222 15223 SDValue SetCC = Ext.getOperand(0); 15224 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse()) 15225 return SDValue(); 15226 15227 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0); 15228 if (CC != X86::COND_E && CC != X86::COND_NE) 15229 return SDValue(); 15230 15231 SDValue Cmp = SetCC.getOperand(1); 15232 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() || 15233 !X86::isZeroNode(Cmp.getOperand(1)) || 15234 !Cmp.getOperand(0).getValueType().isInteger()) 15235 return SDValue(); 15236 15237 SDValue CmpOp0 = Cmp.getOperand(0); 15238 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, 15239 DAG.getConstant(1, CmpOp0.getValueType())); 15240 15241 SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1); 15242 if (CC == X86::COND_NE) 15243 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB, 15244 DL, OtherVal.getValueType(), OtherVal, 15245 DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp); 15246 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC, 15247 DL, OtherVal.getValueType(), OtherVal, 15248 DAG.getConstant(0, OtherVal.getValueType()), NewCmp); 15249} 15250 15251/// PerformADDCombine - Do target-specific dag combines on integer adds. 15252static SDValue PerformAddCombine(SDNode *N, SelectionDAG &DAG, 15253 const X86Subtarget *Subtarget) { 15254 EVT VT = N->getValueType(0); 15255 SDValue Op0 = N->getOperand(0); 15256 SDValue Op1 = N->getOperand(1); 15257 15258 // Try to synthesize horizontal adds from adds of shuffles. 15259 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || 15260 (Subtarget->hasAVX2() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && 15261 isHorizontalBinOp(Op0, Op1, true)) 15262 return DAG.getNode(X86ISD::HADD, N->getDebugLoc(), VT, Op0, Op1); 15263 15264 return OptimizeConditionalInDecrement(N, DAG); 15265} 15266 15267static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG, 15268 const X86Subtarget *Subtarget) { 15269 SDValue Op0 = N->getOperand(0); 15270 SDValue Op1 = N->getOperand(1); 15271 15272 // X86 can't encode an immediate LHS of a sub. See if we can push the 15273 // negation into a preceding instruction. 15274 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) { 15275 // If the RHS of the sub is a XOR with one use and a constant, invert the 15276 // immediate. Then add one to the LHS of the sub so we can turn 15277 // X-Y -> X+~Y+1, saving one register. 15278 if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR && 15279 isa<ConstantSDNode>(Op1.getOperand(1))) { 15280 APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue(); 15281 EVT VT = Op0.getValueType(); 15282 SDValue NewXor = DAG.getNode(ISD::XOR, Op1.getDebugLoc(), VT, 15283 Op1.getOperand(0), 15284 DAG.getConstant(~XorC, VT)); 15285 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, NewXor, 15286 DAG.getConstant(C->getAPIntValue()+1, VT)); 15287 } 15288 } 15289 15290 // Try to synthesize horizontal adds from adds of shuffles. 15291 EVT VT = N->getValueType(0); 15292 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || 15293 (Subtarget->hasAVX2() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && 15294 isHorizontalBinOp(Op0, Op1, true)) 15295 return DAG.getNode(X86ISD::HSUB, N->getDebugLoc(), VT, Op0, Op1); 15296 15297 return OptimizeConditionalInDecrement(N, DAG); 15298} 15299 15300SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 15301 DAGCombinerInfo &DCI) const { 15302 SelectionDAG &DAG = DCI.DAG; 15303 switch (N->getOpcode()) { 15304 default: break; 15305 case ISD::EXTRACT_VECTOR_ELT: 15306 return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, DCI); 15307 case ISD::VSELECT: 15308 case ISD::SELECT: return PerformSELECTCombine(N, DAG, DCI, Subtarget); 15309 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI); 15310 case ISD::ADD: return PerformAddCombine(N, DAG, Subtarget); 15311 case ISD::SUB: return PerformSubCombine(N, DAG, Subtarget); 15312 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI); 15313 case ISD::MUL: return PerformMulCombine(N, DAG, DCI); 15314 case ISD::SHL: 15315 case ISD::SRA: 15316 case ISD::SRL: return PerformShiftCombine(N, DAG, DCI, Subtarget); 15317 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget); 15318 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget); 15319 case ISD::XOR: return PerformXorCombine(N, DAG, DCI, Subtarget); 15320 case ISD::LOAD: return PerformLOADCombine(N, DAG, Subtarget); 15321 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 15322 case ISD::UINT_TO_FP: return PerformUINT_TO_FPCombine(N, DAG); 15323 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, this); 15324 case ISD::FP_TO_SINT: return PerformFP_TO_SINTCombine(N, DAG); 15325 case ISD::FADD: return PerformFADDCombine(N, DAG, Subtarget); 15326 case ISD::FSUB: return PerformFSUBCombine(N, DAG, Subtarget); 15327 case X86ISD::FXOR: 15328 case X86ISD::FOR: return PerformFORCombine(N, DAG); 15329 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 15330 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI); 15331 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG); 15332 case ISD::ANY_EXTEND: 15333 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG, DCI, Subtarget); 15334 case ISD::SIGN_EXTEND: return PerformSExtCombine(N, DAG, DCI, Subtarget); 15335 case ISD::TRUNCATE: return PerformTruncateCombine(N, DAG, DCI); 15336 case ISD::SETCC: return PerformISDSETCCCombine(N, DAG); 15337 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG); 15338 case X86ISD::SHUFP: // Handle all target specific shuffles 15339 case X86ISD::PALIGN: 15340 case X86ISD::UNPCKH: 15341 case X86ISD::UNPCKL: 15342 case X86ISD::MOVHLPS: 15343 case X86ISD::MOVLHPS: 15344 case X86ISD::PSHUFD: 15345 case X86ISD::PSHUFHW: 15346 case X86ISD::PSHUFLW: 15347 case X86ISD::MOVSS: 15348 case X86ISD::MOVSD: 15349 case X86ISD::VPERMILP: 15350 case X86ISD::VPERM2X128: 15351 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget); 15352 } 15353 15354 return SDValue(); 15355} 15356 15357/// isTypeDesirableForOp - Return true if the target has native support for 15358/// the specified value type and it is 'desirable' to use the type for the 15359/// given node type. e.g. On x86 i16 is legal, but undesirable since i16 15360/// instruction encodings are longer and some i16 instructions are slow. 15361bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const { 15362 if (!isTypeLegal(VT)) 15363 return false; 15364 if (VT != MVT::i16) 15365 return true; 15366 15367 switch (Opc) { 15368 default: 15369 return true; 15370 case ISD::LOAD: 15371 case ISD::SIGN_EXTEND: 15372 case ISD::ZERO_EXTEND: 15373 case ISD::ANY_EXTEND: 15374 case ISD::SHL: 15375 case ISD::SRL: 15376 case ISD::SUB: 15377 case ISD::ADD: 15378 case ISD::MUL: 15379 case ISD::AND: 15380 case ISD::OR: 15381 case ISD::XOR: 15382 return false; 15383 } 15384} 15385 15386/// IsDesirableToPromoteOp - This method query the target whether it is 15387/// beneficial for dag combiner to promote the specified node. If true, it 15388/// should return the desired promotion type by reference. 15389bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const { 15390 EVT VT = Op.getValueType(); 15391 if (VT != MVT::i16) 15392 return false; 15393 15394 bool Promote = false; 15395 bool Commute = false; 15396 switch (Op.getOpcode()) { 15397 default: break; 15398 case ISD::LOAD: { 15399 LoadSDNode *LD = cast<LoadSDNode>(Op); 15400 // If the non-extending load has a single use and it's not live out, then it 15401 // might be folded. 15402 if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&& 15403 Op.hasOneUse()*/) { 15404 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 15405 UE = Op.getNode()->use_end(); UI != UE; ++UI) { 15406 // The only case where we'd want to promote LOAD (rather then it being 15407 // promoted as an operand is when it's only use is liveout. 15408 if (UI->getOpcode() != ISD::CopyToReg) 15409 return false; 15410 } 15411 } 15412 Promote = true; 15413 break; 15414 } 15415 case ISD::SIGN_EXTEND: 15416 case ISD::ZERO_EXTEND: 15417 case ISD::ANY_EXTEND: 15418 Promote = true; 15419 break; 15420 case ISD::SHL: 15421 case ISD::SRL: { 15422 SDValue N0 = Op.getOperand(0); 15423 // Look out for (store (shl (load), x)). 15424 if (MayFoldLoad(N0) && MayFoldIntoStore(Op)) 15425 return false; 15426 Promote = true; 15427 break; 15428 } 15429 case ISD::ADD: 15430 case ISD::MUL: 15431 case ISD::AND: 15432 case ISD::OR: 15433 case ISD::XOR: 15434 Commute = true; 15435 // fallthrough 15436 case ISD::SUB: { 15437 SDValue N0 = Op.getOperand(0); 15438 SDValue N1 = Op.getOperand(1); 15439 if (!Commute && MayFoldLoad(N1)) 15440 return false; 15441 // Avoid disabling potential load folding opportunities. 15442 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op))) 15443 return false; 15444 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op))) 15445 return false; 15446 Promote = true; 15447 } 15448 } 15449 15450 PVT = MVT::i32; 15451 return Promote; 15452} 15453 15454//===----------------------------------------------------------------------===// 15455// X86 Inline Assembly Support 15456//===----------------------------------------------------------------------===// 15457 15458namespace { 15459 // Helper to match a string separated by whitespace. 15460 bool matchAsmImpl(StringRef s, ArrayRef<const StringRef *> args) { 15461 s = s.substr(s.find_first_not_of(" \t")); // Skip leading whitespace. 15462 15463 for (unsigned i = 0, e = args.size(); i != e; ++i) { 15464 StringRef piece(*args[i]); 15465 if (!s.startswith(piece)) // Check if the piece matches. 15466 return false; 15467 15468 s = s.substr(piece.size()); 15469 StringRef::size_type pos = s.find_first_not_of(" \t"); 15470 if (pos == 0) // We matched a prefix. 15471 return false; 15472 15473 s = s.substr(pos); 15474 } 15475 15476 return s.empty(); 15477 } 15478 const VariadicFunction1<bool, StringRef, StringRef, matchAsmImpl> matchAsm={}; 15479} 15480 15481bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const { 15482 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); 15483 15484 std::string AsmStr = IA->getAsmString(); 15485 15486 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 15487 if (!Ty || Ty->getBitWidth() % 16 != 0) 15488 return false; 15489 15490 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a" 15491 SmallVector<StringRef, 4> AsmPieces; 15492 SplitString(AsmStr, AsmPieces, ";\n"); 15493 15494 switch (AsmPieces.size()) { 15495 default: return false; 15496 case 1: 15497 // FIXME: this should verify that we are targeting a 486 or better. If not, 15498 // we will turn this bswap into something that will be lowered to logical 15499 // ops instead of emitting the bswap asm. For now, we don't support 486 or 15500 // lower so don't worry about this. 15501 // bswap $0 15502 if (matchAsm(AsmPieces[0], "bswap", "$0") || 15503 matchAsm(AsmPieces[0], "bswapl", "$0") || 15504 matchAsm(AsmPieces[0], "bswapq", "$0") || 15505 matchAsm(AsmPieces[0], "bswap", "${0:q}") || 15506 matchAsm(AsmPieces[0], "bswapl", "${0:q}") || 15507 matchAsm(AsmPieces[0], "bswapq", "${0:q}")) { 15508 // No need to check constraints, nothing other than the equivalent of 15509 // "=r,0" would be valid here. 15510 return IntrinsicLowering::LowerToByteSwap(CI); 15511 } 15512 15513 // rorw $$8, ${0:w} --> llvm.bswap.i16 15514 if (CI->getType()->isIntegerTy(16) && 15515 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 && 15516 (matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") || 15517 matchAsm(AsmPieces[0], "rolw", "$$8,", "${0:w}"))) { 15518 AsmPieces.clear(); 15519 const std::string &ConstraintsStr = IA->getConstraintString(); 15520 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ","); 15521 std::sort(AsmPieces.begin(), AsmPieces.end()); 15522 if (AsmPieces.size() == 4 && 15523 AsmPieces[0] == "~{cc}" && 15524 AsmPieces[1] == "~{dirflag}" && 15525 AsmPieces[2] == "~{flags}" && 15526 AsmPieces[3] == "~{fpsr}") 15527 return IntrinsicLowering::LowerToByteSwap(CI); 15528 } 15529 break; 15530 case 3: 15531 if (CI->getType()->isIntegerTy(32) && 15532 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 && 15533 matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") && 15534 matchAsm(AsmPieces[1], "rorl", "$$16,", "$0") && 15535 matchAsm(AsmPieces[2], "rorw", "$$8,", "${0:w}")) { 15536 AsmPieces.clear(); 15537 const std::string &ConstraintsStr = IA->getConstraintString(); 15538 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ","); 15539 std::sort(AsmPieces.begin(), AsmPieces.end()); 15540 if (AsmPieces.size() == 4 && 15541 AsmPieces[0] == "~{cc}" && 15542 AsmPieces[1] == "~{dirflag}" && 15543 AsmPieces[2] == "~{flags}" && 15544 AsmPieces[3] == "~{fpsr}") 15545 return IntrinsicLowering::LowerToByteSwap(CI); 15546 } 15547 15548 if (CI->getType()->isIntegerTy(64)) { 15549 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints(); 15550 if (Constraints.size() >= 2 && 15551 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" && 15552 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") { 15553 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64 15554 if (matchAsm(AsmPieces[0], "bswap", "%eax") && 15555 matchAsm(AsmPieces[1], "bswap", "%edx") && 15556 matchAsm(AsmPieces[2], "xchgl", "%eax,", "%edx")) 15557 return IntrinsicLowering::LowerToByteSwap(CI); 15558 } 15559 } 15560 break; 15561 } 15562 return false; 15563} 15564 15565 15566 15567/// getConstraintType - Given a constraint letter, return the type of 15568/// constraint it is for this target. 15569X86TargetLowering::ConstraintType 15570X86TargetLowering::getConstraintType(const std::string &Constraint) const { 15571 if (Constraint.size() == 1) { 15572 switch (Constraint[0]) { 15573 case 'R': 15574 case 'q': 15575 case 'Q': 15576 case 'f': 15577 case 't': 15578 case 'u': 15579 case 'y': 15580 case 'x': 15581 case 'Y': 15582 case 'l': 15583 return C_RegisterClass; 15584 case 'a': 15585 case 'b': 15586 case 'c': 15587 case 'd': 15588 case 'S': 15589 case 'D': 15590 case 'A': 15591 return C_Register; 15592 case 'I': 15593 case 'J': 15594 case 'K': 15595 case 'L': 15596 case 'M': 15597 case 'N': 15598 case 'G': 15599 case 'C': 15600 case 'e': 15601 case 'Z': 15602 return C_Other; 15603 default: 15604 break; 15605 } 15606 } 15607 return TargetLowering::getConstraintType(Constraint); 15608} 15609 15610/// Examine constraint type and operand type and determine a weight value. 15611/// This object must already have been set up with the operand type 15612/// and the current alternative constraint selected. 15613TargetLowering::ConstraintWeight 15614 X86TargetLowering::getSingleConstraintMatchWeight( 15615 AsmOperandInfo &info, const char *constraint) const { 15616 ConstraintWeight weight = CW_Invalid; 15617 Value *CallOperandVal = info.CallOperandVal; 15618 // If we don't have a value, we can't do a match, 15619 // but allow it at the lowest weight. 15620 if (CallOperandVal == NULL) 15621 return CW_Default; 15622 Type *type = CallOperandVal->getType(); 15623 // Look at the constraint type. 15624 switch (*constraint) { 15625 default: 15626 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 15627 case 'R': 15628 case 'q': 15629 case 'Q': 15630 case 'a': 15631 case 'b': 15632 case 'c': 15633 case 'd': 15634 case 'S': 15635 case 'D': 15636 case 'A': 15637 if (CallOperandVal->getType()->isIntegerTy()) 15638 weight = CW_SpecificReg; 15639 break; 15640 case 'f': 15641 case 't': 15642 case 'u': 15643 if (type->isFloatingPointTy()) 15644 weight = CW_SpecificReg; 15645 break; 15646 case 'y': 15647 if (type->isX86_MMXTy() && Subtarget->hasMMX()) 15648 weight = CW_SpecificReg; 15649 break; 15650 case 'x': 15651 case 'Y': 15652 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget->hasSSE1()) || 15653 ((type->getPrimitiveSizeInBits() == 256) && Subtarget->hasAVX())) 15654 weight = CW_Register; 15655 break; 15656 case 'I': 15657 if (ConstantInt *C = dyn_cast<ConstantInt>(info.CallOperandVal)) { 15658 if (C->getZExtValue() <= 31) 15659 weight = CW_Constant; 15660 } 15661 break; 15662 case 'J': 15663 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15664 if (C->getZExtValue() <= 63) 15665 weight = CW_Constant; 15666 } 15667 break; 15668 case 'K': 15669 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15670 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f)) 15671 weight = CW_Constant; 15672 } 15673 break; 15674 case 'L': 15675 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15676 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff)) 15677 weight = CW_Constant; 15678 } 15679 break; 15680 case 'M': 15681 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15682 if (C->getZExtValue() <= 3) 15683 weight = CW_Constant; 15684 } 15685 break; 15686 case 'N': 15687 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15688 if (C->getZExtValue() <= 0xff) 15689 weight = CW_Constant; 15690 } 15691 break; 15692 case 'G': 15693 case 'C': 15694 if (dyn_cast<ConstantFP>(CallOperandVal)) { 15695 weight = CW_Constant; 15696 } 15697 break; 15698 case 'e': 15699 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15700 if ((C->getSExtValue() >= -0x80000000LL) && 15701 (C->getSExtValue() <= 0x7fffffffLL)) 15702 weight = CW_Constant; 15703 } 15704 break; 15705 case 'Z': 15706 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) { 15707 if (C->getZExtValue() <= 0xffffffff) 15708 weight = CW_Constant; 15709 } 15710 break; 15711 } 15712 return weight; 15713} 15714 15715/// LowerXConstraint - try to replace an X constraint, which matches anything, 15716/// with another that has more specific requirements based on the type of the 15717/// corresponding operand. 15718const char *X86TargetLowering:: 15719LowerXConstraint(EVT ConstraintVT) const { 15720 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 15721 // 'f' like normal targets. 15722 if (ConstraintVT.isFloatingPoint()) { 15723 if (Subtarget->hasSSE2()) 15724 return "Y"; 15725 if (Subtarget->hasSSE1()) 15726 return "x"; 15727 } 15728 15729 return TargetLowering::LowerXConstraint(ConstraintVT); 15730} 15731 15732/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 15733/// vector. If it is invalid, don't add anything to Ops. 15734void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 15735 std::string &Constraint, 15736 std::vector<SDValue>&Ops, 15737 SelectionDAG &DAG) const { 15738 SDValue Result(0, 0); 15739 15740 // Only support length 1 constraints for now. 15741 if (Constraint.length() > 1) return; 15742 15743 char ConstraintLetter = Constraint[0]; 15744 switch (ConstraintLetter) { 15745 default: break; 15746 case 'I': 15747 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15748 if (C->getZExtValue() <= 31) { 15749 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 15750 break; 15751 } 15752 } 15753 return; 15754 case 'J': 15755 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15756 if (C->getZExtValue() <= 63) { 15757 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 15758 break; 15759 } 15760 } 15761 return; 15762 case 'K': 15763 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15764 if ((int8_t)C->getSExtValue() == C->getSExtValue()) { 15765 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 15766 break; 15767 } 15768 } 15769 return; 15770 case 'N': 15771 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15772 if (C->getZExtValue() <= 255) { 15773 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 15774 break; 15775 } 15776 } 15777 return; 15778 case 'e': { 15779 // 32-bit signed value 15780 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15781 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 15782 C->getSExtValue())) { 15783 // Widen to 64 bits here to get it sign extended. 15784 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); 15785 break; 15786 } 15787 // FIXME gcc accepts some relocatable values here too, but only in certain 15788 // memory models; it's complicated. 15789 } 15790 return; 15791 } 15792 case 'Z': { 15793 // 32-bit unsigned value 15794 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 15795 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 15796 C->getZExtValue())) { 15797 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 15798 break; 15799 } 15800 } 15801 // FIXME gcc accepts some relocatable values here too, but only in certain 15802 // memory models; it's complicated. 15803 return; 15804 } 15805 case 'i': { 15806 // Literal immediates are always ok. 15807 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 15808 // Widen to 64 bits here to get it sign extended. 15809 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); 15810 break; 15811 } 15812 15813 // In any sort of PIC mode addresses need to be computed at runtime by 15814 // adding in a register or some sort of table lookup. These can't 15815 // be used as immediates. 15816 if (Subtarget->isPICStyleGOT() || Subtarget->isPICStyleStubPIC()) 15817 return; 15818 15819 // If we are in non-pic codegen mode, we allow the address of a global (with 15820 // an optional displacement) to be used with 'i'. 15821 GlobalAddressSDNode *GA = 0; 15822 int64_t Offset = 0; 15823 15824 // Match either (GA), (GA+C), (GA+C1+C2), etc. 15825 while (1) { 15826 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) { 15827 Offset += GA->getOffset(); 15828 break; 15829 } else if (Op.getOpcode() == ISD::ADD) { 15830 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 15831 Offset += C->getZExtValue(); 15832 Op = Op.getOperand(0); 15833 continue; 15834 } 15835 } else if (Op.getOpcode() == ISD::SUB) { 15836 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 15837 Offset += -C->getZExtValue(); 15838 Op = Op.getOperand(0); 15839 continue; 15840 } 15841 } 15842 15843 // Otherwise, this isn't something we can handle, reject it. 15844 return; 15845 } 15846 15847 const GlobalValue *GV = GA->getGlobal(); 15848 // If we require an extra load to get this address, as in PIC mode, we 15849 // can't accept it. 15850 if (isGlobalStubReference(Subtarget->ClassifyGlobalReference(GV, 15851 getTargetMachine()))) 15852 return; 15853 15854 Result = DAG.getTargetGlobalAddress(GV, Op.getDebugLoc(), 15855 GA->getValueType(0), Offset); 15856 break; 15857 } 15858 } 15859 15860 if (Result.getNode()) { 15861 Ops.push_back(Result); 15862 return; 15863 } 15864 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 15865} 15866 15867std::pair<unsigned, const TargetRegisterClass*> 15868X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 15869 EVT VT) const { 15870 // First, see if this is a constraint that directly corresponds to an LLVM 15871 // register class. 15872 if (Constraint.size() == 1) { 15873 // GCC Constraint Letters 15874 switch (Constraint[0]) { 15875 default: break; 15876 // TODO: Slight differences here in allocation order and leaving 15877 // RIP in the class. Do they matter any more here than they do 15878 // in the normal allocation? 15879 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode. 15880 if (Subtarget->is64Bit()) { 15881 if (VT == MVT::i32 || VT == MVT::f32) 15882 return std::make_pair(0U, &X86::GR32RegClass); 15883 if (VT == MVT::i16) 15884 return std::make_pair(0U, &X86::GR16RegClass); 15885 if (VT == MVT::i8 || VT == MVT::i1) 15886 return std::make_pair(0U, &X86::GR8RegClass); 15887 if (VT == MVT::i64 || VT == MVT::f64) 15888 return std::make_pair(0U, &X86::GR64RegClass); 15889 break; 15890 } 15891 // 32-bit fallthrough 15892 case 'Q': // Q_REGS 15893 if (VT == MVT::i32 || VT == MVT::f32) 15894 return std::make_pair(0U, &X86::GR32_ABCDRegClass); 15895 if (VT == MVT::i16) 15896 return std::make_pair(0U, &X86::GR16_ABCDRegClass); 15897 if (VT == MVT::i8 || VT == MVT::i1) 15898 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass); 15899 if (VT == MVT::i64) 15900 return std::make_pair(0U, &X86::GR64_ABCDRegClass); 15901 break; 15902 case 'r': // GENERAL_REGS 15903 case 'l': // INDEX_REGS 15904 if (VT == MVT::i8 || VT == MVT::i1) 15905 return std::make_pair(0U, &X86::GR8RegClass); 15906 if (VT == MVT::i16) 15907 return std::make_pair(0U, &X86::GR16RegClass); 15908 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit()) 15909 return std::make_pair(0U, &X86::GR32RegClass); 15910 return std::make_pair(0U, &X86::GR64RegClass); 15911 case 'R': // LEGACY_REGS 15912 if (VT == MVT::i8 || VT == MVT::i1) 15913 return std::make_pair(0U, &X86::GR8_NOREXRegClass); 15914 if (VT == MVT::i16) 15915 return std::make_pair(0U, &X86::GR16_NOREXRegClass); 15916 if (VT == MVT::i32 || !Subtarget->is64Bit()) 15917 return std::make_pair(0U, &X86::GR32_NOREXRegClass); 15918 return std::make_pair(0U, &X86::GR64_NOREXRegClass); 15919 case 'f': // FP Stack registers. 15920 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 15921 // value to the correct fpstack register class. 15922 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 15923 return std::make_pair(0U, &X86::RFP32RegClass); 15924 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 15925 return std::make_pair(0U, &X86::RFP64RegClass); 15926 return std::make_pair(0U, &X86::RFP80RegClass); 15927 case 'y': // MMX_REGS if MMX allowed. 15928 if (!Subtarget->hasMMX()) break; 15929 return std::make_pair(0U, &X86::VR64RegClass); 15930 case 'Y': // SSE_REGS if SSE2 allowed 15931 if (!Subtarget->hasSSE2()) break; 15932 // FALL THROUGH. 15933 case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed 15934 if (!Subtarget->hasSSE1()) break; 15935 15936 switch (VT.getSimpleVT().SimpleTy) { 15937 default: break; 15938 // Scalar SSE types. 15939 case MVT::f32: 15940 case MVT::i32: 15941 return std::make_pair(0U, &X86::FR32RegClass); 15942 case MVT::f64: 15943 case MVT::i64: 15944 return std::make_pair(0U, &X86::FR64RegClass); 15945 // Vector types. 15946 case MVT::v16i8: 15947 case MVT::v8i16: 15948 case MVT::v4i32: 15949 case MVT::v2i64: 15950 case MVT::v4f32: 15951 case MVT::v2f64: 15952 return std::make_pair(0U, &X86::VR128RegClass); 15953 // AVX types. 15954 case MVT::v32i8: 15955 case MVT::v16i16: 15956 case MVT::v8i32: 15957 case MVT::v4i64: 15958 case MVT::v8f32: 15959 case MVT::v4f64: 15960 return std::make_pair(0U, &X86::VR256RegClass); 15961 } 15962 break; 15963 } 15964 } 15965 15966 // Use the default implementation in TargetLowering to convert the register 15967 // constraint into a member of a register class. 15968 std::pair<unsigned, const TargetRegisterClass*> Res; 15969 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 15970 15971 // Not found as a standard register? 15972 if (Res.second == 0) { 15973 // Map st(0) -> st(7) -> ST0 15974 if (Constraint.size() == 7 && Constraint[0] == '{' && 15975 tolower(Constraint[1]) == 's' && 15976 tolower(Constraint[2]) == 't' && 15977 Constraint[3] == '(' && 15978 (Constraint[4] >= '0' && Constraint[4] <= '7') && 15979 Constraint[5] == ')' && 15980 Constraint[6] == '}') { 15981 15982 Res.first = X86::ST0+Constraint[4]-'0'; 15983 Res.second = &X86::RFP80RegClass; 15984 return Res; 15985 } 15986 15987 // GCC allows "st(0)" to be called just plain "st". 15988 if (StringRef("{st}").equals_lower(Constraint)) { 15989 Res.first = X86::ST0; 15990 Res.second = &X86::RFP80RegClass; 15991 return Res; 15992 } 15993 15994 // flags -> EFLAGS 15995 if (StringRef("{flags}").equals_lower(Constraint)) { 15996 Res.first = X86::EFLAGS; 15997 Res.second = &X86::CCRRegClass; 15998 return Res; 15999 } 16000 16001 // 'A' means EAX + EDX. 16002 if (Constraint == "A") { 16003 Res.first = X86::EAX; 16004 Res.second = &X86::GR32_ADRegClass; 16005 return Res; 16006 } 16007 return Res; 16008 } 16009 16010 // Otherwise, check to see if this is a register class of the wrong value 16011 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 16012 // turn into {ax},{dx}. 16013 if (Res.second->hasType(VT)) 16014 return Res; // Correct type already, nothing to do. 16015 16016 // All of the single-register GCC register classes map their values onto 16017 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 16018 // really want an 8-bit or 32-bit register, map to the appropriate register 16019 // class and return the appropriate register. 16020 if (Res.second == &X86::GR16RegClass) { 16021 if (VT == MVT::i8) { 16022 unsigned DestReg = 0; 16023 switch (Res.first) { 16024 default: break; 16025 case X86::AX: DestReg = X86::AL; break; 16026 case X86::DX: DestReg = X86::DL; break; 16027 case X86::CX: DestReg = X86::CL; break; 16028 case X86::BX: DestReg = X86::BL; break; 16029 } 16030 if (DestReg) { 16031 Res.first = DestReg; 16032 Res.second = &X86::GR8RegClass; 16033 } 16034 } else if (VT == MVT::i32) { 16035 unsigned DestReg = 0; 16036 switch (Res.first) { 16037 default: break; 16038 case X86::AX: DestReg = X86::EAX; break; 16039 case X86::DX: DestReg = X86::EDX; break; 16040 case X86::CX: DestReg = X86::ECX; break; 16041 case X86::BX: DestReg = X86::EBX; break; 16042 case X86::SI: DestReg = X86::ESI; break; 16043 case X86::DI: DestReg = X86::EDI; break; 16044 case X86::BP: DestReg = X86::EBP; break; 16045 case X86::SP: DestReg = X86::ESP; break; 16046 } 16047 if (DestReg) { 16048 Res.first = DestReg; 16049 Res.second = &X86::GR32RegClass; 16050 } 16051 } else if (VT == MVT::i64) { 16052 unsigned DestReg = 0; 16053 switch (Res.first) { 16054 default: break; 16055 case X86::AX: DestReg = X86::RAX; break; 16056 case X86::DX: DestReg = X86::RDX; break; 16057 case X86::CX: DestReg = X86::RCX; break; 16058 case X86::BX: DestReg = X86::RBX; break; 16059 case X86::SI: DestReg = X86::RSI; break; 16060 case X86::DI: DestReg = X86::RDI; break; 16061 case X86::BP: DestReg = X86::RBP; break; 16062 case X86::SP: DestReg = X86::RSP; break; 16063 } 16064 if (DestReg) { 16065 Res.first = DestReg; 16066 Res.second = &X86::GR64RegClass; 16067 } 16068 } 16069 } else if (Res.second == &X86::FR32RegClass || 16070 Res.second == &X86::FR64RegClass || 16071 Res.second == &X86::VR128RegClass) { 16072 // Handle references to XMM physical registers that got mapped into the 16073 // wrong class. This can happen with constraints like {xmm0} where the 16074 // target independent register mapper will just pick the first match it can 16075 // find, ignoring the required type. 16076 if (VT == MVT::f32) 16077 Res.second = &X86::FR32RegClass; 16078 else if (VT == MVT::f64) 16079 Res.second = &X86::FR64RegClass; 16080 else if (X86::VR128RegClass.hasType(VT)) 16081 Res.second = &X86::VR128RegClass; 16082 } 16083 16084 return Res; 16085} 16086