X86ISelLowering.cpp revision f595266b91ea5aef953ce3019557a5f3d3291ff9
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86TargetMachine.h" 20#include "llvm/CallingConv.h" 21#include "llvm/Constants.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/GlobalVariable.h" 24#include "llvm/Function.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/ADT/BitVector.h" 27#include "llvm/ADT/VectorExtras.h" 28#include "llvm/CodeGen/CallingConvLower.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/PseudoSourceValue.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/Debug.h" 38#include "llvm/Target/TargetOptions.h" 39#include "llvm/ADT/SmallSet.h" 40#include "llvm/ADT/StringExtras.h" 41using namespace llvm; 42 43// Forward declarations. 44static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45 46X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47 : TargetLowering(TM) { 48 Subtarget = &TM.getSubtarget<X86Subtarget>(); 49 X86ScalarSSEf64 = Subtarget->hasSSE2(); 50 X86ScalarSSEf32 = Subtarget->hasSSE1(); 51 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52 53 bool Fast = false; 54 55 RegInfo = TM.getRegisterInfo(); 56 57 // Set up the TargetLowering object. 58 59 // X86 is weird, it always uses i8 for shift amounts and setcc results. 60 setShiftAmountType(MVT::i8); 61 setSetCCResultContents(ZeroOrOneSetCCResult); 62 setSchedulingPreference(SchedulingForRegPressure); 63 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 64 setStackPointerRegisterToSaveRestore(X86StackPtr); 65 66 if (Subtarget->isTargetDarwin()) { 67 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 68 setUseUnderscoreSetJmp(false); 69 setUseUnderscoreLongJmp(false); 70 } else if (Subtarget->isTargetMingw()) { 71 // MS runtime is weird: it exports _setjmp, but longjmp! 72 setUseUnderscoreSetJmp(true); 73 setUseUnderscoreLongJmp(false); 74 } else { 75 setUseUnderscoreSetJmp(true); 76 setUseUnderscoreLongJmp(true); 77 } 78 79 // Set up the register classes. 80 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 81 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 82 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 83 if (Subtarget->is64Bit()) 84 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 85 86 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 87 88 // We don't accept any truncstore of integer registers. 89 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 90 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 91 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 92 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 93 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 94 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 95 96 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 97 // operation. 98 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 99 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 100 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 101 102 if (Subtarget->is64Bit()) { 103 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 104 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 105 } else { 106 if (X86ScalarSSEf64) 107 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 108 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 109 else 110 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 111 } 112 113 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 114 // this operation. 115 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 116 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 117 // SSE has no i16 to fp conversion, only i32 118 if (X86ScalarSSEf32) { 119 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 120 // f32 and f64 cases are Legal, f80 case is not 121 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 122 } else { 123 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 124 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 125 } 126 127 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 128 // are Legal, f80 is custom lowered. 129 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 130 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 131 132 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 133 // this operation. 134 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 135 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 136 137 if (X86ScalarSSEf32) { 138 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 139 // f32 and f64 cases are Legal, f80 case is not 140 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 141 } else { 142 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 143 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 144 } 145 146 // Handle FP_TO_UINT by promoting the destination to a larger signed 147 // conversion. 148 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 149 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 151 152 if (Subtarget->is64Bit()) { 153 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 155 } else { 156 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 157 // Expand FP_TO_UINT into a select. 158 // FIXME: We would like to use a Custom expander here eventually to do 159 // the optimal thing for SSE vs. the default expansion in the legalizer. 160 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 161 else 162 // With SSE3 we can use fisttpll to convert to a signed i64. 163 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 164 } 165 166 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 167 if (!X86ScalarSSEf64) { 168 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 169 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 170 } 171 172 // Scalar integer divide and remainder are lowered to use operations that 173 // produce two results, to match the available instructions. This exposes 174 // the two-result form to trivial CSE, which is able to combine x/y and x%y 175 // into a single instruction. 176 // 177 // Scalar integer multiply-high is also lowered to use two-result 178 // operations, to match the available instructions. However, plain multiply 179 // (low) operations are left as Legal, as there are single-result 180 // instructions for this in x86. Using the two-result multiply instructions 181 // when both high and low results are needed must be arranged by dagcombine. 182 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 183 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 184 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 185 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::SREM , MVT::i8 , Expand); 187 setOperationAction(ISD::UREM , MVT::i8 , Expand); 188 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 189 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 190 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 191 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::SREM , MVT::i16 , Expand); 193 setOperationAction(ISD::UREM , MVT::i16 , Expand); 194 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 195 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 196 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 197 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::SREM , MVT::i32 , Expand); 199 setOperationAction(ISD::UREM , MVT::i32 , Expand); 200 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 201 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 202 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 203 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::SREM , MVT::i64 , Expand); 205 setOperationAction(ISD::UREM , MVT::i64 , Expand); 206 207 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 208 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 209 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 210 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 211 if (Subtarget->is64Bit()) 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 216 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 217 setOperationAction(ISD::FREM , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f64 , Expand); 219 setOperationAction(ISD::FREM , MVT::f80 , Expand); 220 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 221 222 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 223 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 224 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 226 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 227 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 229 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 230 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 231 if (Subtarget->is64Bit()) { 232 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 233 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 234 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 235 } 236 237 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 238 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 239 240 // These should be promoted to a larger select which is supported. 241 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 242 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 243 // X86 wants to expand cmov itself. 244 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 245 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 246 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 249 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 252 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 255 if (Subtarget->is64Bit()) { 256 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 257 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 258 } 259 // X86 ret instruction may pop stack. 260 setOperationAction(ISD::RET , MVT::Other, Custom); 261 if (!Subtarget->is64Bit()) 262 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 263 264 // Darwin ABI issue. 265 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 266 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 268 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 269 if (Subtarget->is64Bit()) 270 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 271 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 272 if (Subtarget->is64Bit()) { 273 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 274 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 275 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 276 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 277 } 278 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 279 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 281 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 282 if (Subtarget->is64Bit()) { 283 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 285 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 286 } 287 288 if (Subtarget->hasSSE1()) 289 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 290 291 if (!Subtarget->hasSSE2()) 292 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 293 294 // Expand certain atomics 295 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i8, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i16, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i32, Custom); 298 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i64, Custom); 299 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i32, Expand); 300 301 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 302 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 303 // FIXME - use subtarget debug flags 304 if (!Subtarget->isTargetDarwin() && 305 !Subtarget->isTargetELF() && 306 !Subtarget->isTargetCygMing()) { 307 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 308 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 309 } 310 311 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 312 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 313 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 314 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 315 if (Subtarget->is64Bit()) { 316 // FIXME: Verify 317 setExceptionPointerRegister(X86::RAX); 318 setExceptionSelectorRegister(X86::RDX); 319 } else { 320 setExceptionPointerRegister(X86::EAX); 321 setExceptionSelectorRegister(X86::EDX); 322 } 323 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 324 325 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 326 327 setOperationAction(ISD::TRAP, MVT::Other, Legal); 328 329 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 330 setOperationAction(ISD::VASTART , MVT::Other, Custom); 331 setOperationAction(ISD::VAEND , MVT::Other, Expand); 332 if (Subtarget->is64Bit()) { 333 setOperationAction(ISD::VAARG , MVT::Other, Custom); 334 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 335 } else { 336 setOperationAction(ISD::VAARG , MVT::Other, Expand); 337 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 338 } 339 340 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 341 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 342 if (Subtarget->is64Bit()) 343 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 344 if (Subtarget->isTargetCygMing()) 345 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 346 else 347 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 348 349 if (X86ScalarSSEf64) { 350 // f32 and f64 use SSE. 351 // Set up the FP register classes. 352 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 353 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 354 355 // Use ANDPD to simulate FABS. 356 setOperationAction(ISD::FABS , MVT::f64, Custom); 357 setOperationAction(ISD::FABS , MVT::f32, Custom); 358 359 // Use XORP to simulate FNEG. 360 setOperationAction(ISD::FNEG , MVT::f64, Custom); 361 setOperationAction(ISD::FNEG , MVT::f32, Custom); 362 363 // Use ANDPD and ORPD to simulate FCOPYSIGN. 364 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 365 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 366 367 // We don't support sin/cos/fmod 368 setOperationAction(ISD::FSIN , MVT::f64, Expand); 369 setOperationAction(ISD::FCOS , MVT::f64, Expand); 370 setOperationAction(ISD::FSIN , MVT::f32, Expand); 371 setOperationAction(ISD::FCOS , MVT::f32, Expand); 372 373 // Expand FP immediates into loads from the stack, except for the special 374 // cases we handle. 375 addLegalFPImmediate(APFloat(+0.0)); // xorpd 376 addLegalFPImmediate(APFloat(+0.0f)); // xorps 377 378 // Floating truncations from f80 and extensions to f80 go through memory. 379 // If optimizing, we lie about this though and handle it in 380 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 381 if (Fast) { 382 setConvertAction(MVT::f32, MVT::f80, Expand); 383 setConvertAction(MVT::f64, MVT::f80, Expand); 384 setConvertAction(MVT::f80, MVT::f32, Expand); 385 setConvertAction(MVT::f80, MVT::f64, Expand); 386 } 387 } else if (X86ScalarSSEf32) { 388 // Use SSE for f32, x87 for f64. 389 // Set up the FP register classes. 390 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 391 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 392 393 // Use ANDPS to simulate FABS. 394 setOperationAction(ISD::FABS , MVT::f32, Custom); 395 396 // Use XORP to simulate FNEG. 397 setOperationAction(ISD::FNEG , MVT::f32, Custom); 398 399 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 400 401 // Use ANDPS and ORPS to simulate FCOPYSIGN. 402 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 403 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 404 405 // We don't support sin/cos/fmod 406 setOperationAction(ISD::FSIN , MVT::f32, Expand); 407 setOperationAction(ISD::FCOS , MVT::f32, Expand); 408 409 // Special cases we handle for FP constants. 410 addLegalFPImmediate(APFloat(+0.0f)); // xorps 411 addLegalFPImmediate(APFloat(+0.0)); // FLD0 412 addLegalFPImmediate(APFloat(+1.0)); // FLD1 413 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 414 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 415 416 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 417 // this though and handle it in InstructionSelectPreprocess so that 418 // dagcombine2 can hack on these. 419 if (Fast) { 420 setConvertAction(MVT::f32, MVT::f64, Expand); 421 setConvertAction(MVT::f32, MVT::f80, Expand); 422 setConvertAction(MVT::f80, MVT::f32, Expand); 423 setConvertAction(MVT::f64, MVT::f32, Expand); 424 // And x87->x87 truncations also. 425 setConvertAction(MVT::f80, MVT::f64, Expand); 426 } 427 428 if (!UnsafeFPMath) { 429 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 430 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 431 } 432 } else { 433 // f32 and f64 in x87. 434 // Set up the FP register classes. 435 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 436 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 437 438 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 439 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 440 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 441 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 442 443 // Floating truncations go through memory. If optimizing, we lie about 444 // this though and handle it in InstructionSelectPreprocess so that 445 // dagcombine2 can hack on these. 446 if (Fast) { 447 setConvertAction(MVT::f80, MVT::f32, Expand); 448 setConvertAction(MVT::f64, MVT::f32, Expand); 449 setConvertAction(MVT::f80, MVT::f64, Expand); 450 } 451 452 if (!UnsafeFPMath) { 453 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 454 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 455 } 456 addLegalFPImmediate(APFloat(+0.0)); // FLD0 457 addLegalFPImmediate(APFloat(+1.0)); // FLD1 458 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 459 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 460 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 461 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 462 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 463 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 464 } 465 466 // Long double always uses X87. 467 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 468 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 469 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 470 { 471 APFloat TmpFlt(+0.0); 472 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 473 addLegalFPImmediate(TmpFlt); // FLD0 474 TmpFlt.changeSign(); 475 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 476 APFloat TmpFlt2(+1.0); 477 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 478 addLegalFPImmediate(TmpFlt2); // FLD1 479 TmpFlt2.changeSign(); 480 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 481 } 482 483 if (!UnsafeFPMath) { 484 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 485 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 486 } 487 488 // Always use a library call for pow. 489 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 490 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 491 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 492 493 // First set operation action for all vector types to expand. Then we 494 // will selectively turn on ones that can be effectively codegen'd. 495 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 496 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 497 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 498 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 499 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 500 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 501 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 502 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 503 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 504 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 505 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 506 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 507 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 508 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 509 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 510 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 511 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 512 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 513 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 514 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 515 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 516 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 517 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 532 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 533 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 534 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 535 } 536 537 if (Subtarget->hasMMX()) { 538 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 539 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 540 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 541 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 542 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 543 544 // FIXME: add MMX packed arithmetics 545 546 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 547 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 548 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 549 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 550 551 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 552 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 553 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 554 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 555 556 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 557 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 558 559 setOperationAction(ISD::AND, MVT::v8i8, Promote); 560 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 561 setOperationAction(ISD::AND, MVT::v4i16, Promote); 562 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 563 setOperationAction(ISD::AND, MVT::v2i32, Promote); 564 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 565 setOperationAction(ISD::AND, MVT::v1i64, Legal); 566 567 setOperationAction(ISD::OR, MVT::v8i8, Promote); 568 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 569 setOperationAction(ISD::OR, MVT::v4i16, Promote); 570 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 571 setOperationAction(ISD::OR, MVT::v2i32, Promote); 572 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 573 setOperationAction(ISD::OR, MVT::v1i64, Legal); 574 575 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 576 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 577 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 578 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 579 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 580 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 581 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 582 583 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 584 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 585 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 586 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 587 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 588 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 589 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 590 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 591 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 592 593 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 594 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 595 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 596 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 597 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 598 599 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 600 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 601 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 602 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 603 604 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 605 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 606 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 607 } 608 609 if (Subtarget->hasSSE1()) { 610 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 611 612 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 613 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 614 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 615 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 616 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 617 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 618 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 619 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 620 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 621 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 622 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 623 setOperationAction(ISD::VSETCC, MVT::v4f32, Legal); 624 } 625 626 if (Subtarget->hasSSE2()) { 627 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 628 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 629 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 630 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 631 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 632 633 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 634 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 635 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 636 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 637 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 638 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 639 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 640 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 641 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 642 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 643 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 644 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 645 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 646 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 647 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 648 649 setOperationAction(ISD::VSETCC, MVT::v2f64, Legal); 650 setOperationAction(ISD::VSETCC, MVT::v16i8, Legal); 651 setOperationAction(ISD::VSETCC, MVT::v8i16, Legal); 652 setOperationAction(ISD::VSETCC, MVT::v4i32, Legal); 653 setOperationAction(ISD::VSETCC, MVT::v2i64, Legal); 654 655 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 656 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 657 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 658 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 659 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 660 661 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 662 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 663 MVT VT = (MVT::SimpleValueType)i; 664 // Do not attempt to custom lower non-power-of-2 vectors 665 if (!isPowerOf2_32(VT.getVectorNumElements())) 666 continue; 667 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 668 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 669 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 670 } 671 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 672 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 673 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 674 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 675 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 676 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 677 if (Subtarget->is64Bit()) { 678 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 679 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 680 } 681 682 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 683 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 684 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 685 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 686 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 687 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 688 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 689 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 690 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 691 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 692 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 693 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 694 } 695 696 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 697 698 // Custom lower v2i64 and v2f64 selects. 699 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 700 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 701 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 702 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 703 704 } 705 706 if (Subtarget->hasSSE41()) { 707 // FIXME: Do we need to handle scalar-to-vector here? 708 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 709 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 710 711 // i8 and i16 vectors are custom , because the source register and source 712 // source memory operand types are not the same width. f32 vectors are 713 // custom since the immediate controlling the insert encodes additional 714 // information. 715 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 716 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 717 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 718 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 719 720 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 721 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 722 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 723 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 724 725 if (Subtarget->is64Bit()) { 726 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 727 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 728 } 729 } 730 731 // We want to custom lower some of our intrinsics. 732 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 733 734 // We have target-specific dag combine patterns for the following nodes: 735 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 736 setTargetDAGCombine(ISD::BUILD_VECTOR); 737 setTargetDAGCombine(ISD::SELECT); 738 setTargetDAGCombine(ISD::STORE); 739 740 computeRegisterProperties(); 741 742 // FIXME: These should be based on subtarget info. Plus, the values should 743 // be smaller when we are in optimizing for size mode. 744 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 745 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 746 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 747 allowUnalignedMemoryAccesses = true; // x86 supports it! 748 setPrefLoopAlignment(16); 749} 750 751 752MVT X86TargetLowering::getSetCCResultType(const SDOperand &) const { 753 return MVT::i8; 754} 755 756 757/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 758/// the desired ByVal argument alignment. 759static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 760 if (MaxAlign == 16) 761 return; 762 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 763 if (VTy->getBitWidth() == 128) 764 MaxAlign = 16; 765 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 766 unsigned EltAlign = 0; 767 getMaxByValAlign(ATy->getElementType(), EltAlign); 768 if (EltAlign > MaxAlign) 769 MaxAlign = EltAlign; 770 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 771 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 772 unsigned EltAlign = 0; 773 getMaxByValAlign(STy->getElementType(i), EltAlign); 774 if (EltAlign > MaxAlign) 775 MaxAlign = EltAlign; 776 if (MaxAlign == 16) 777 break; 778 } 779 } 780 return; 781} 782 783/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 784/// function arguments in the caller parameter area. For X86, aggregates 785/// that contain SSE vectors are placed at 16-byte boundaries while the rest 786/// are at 4-byte boundaries. 787unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 788 if (Subtarget->is64Bit()) 789 return getTargetData()->getABITypeAlignment(Ty); 790 unsigned Align = 4; 791 if (Subtarget->hasSSE1()) 792 getMaxByValAlign(Ty, Align); 793 return Align; 794} 795 796/// getOptimalMemOpType - Returns the target specific optimal type for load 797/// and store operations as a result of memset, memcpy, and memmove 798/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 799/// determining it. 800MVT 801X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 802 bool isSrcConst, bool isSrcStr) const { 803 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 804 return MVT::v4i32; 805 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 806 return MVT::v4f32; 807 if (Subtarget->is64Bit() && Size >= 8) 808 return MVT::i64; 809 return MVT::i32; 810} 811 812 813/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 814/// jumptable. 815SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table, 816 SelectionDAG &DAG) const { 817 if (usesGlobalOffsetTable()) 818 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 819 if (!Subtarget->isPICStyleRIPRel()) 820 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 821 return Table; 822} 823 824//===----------------------------------------------------------------------===// 825// Return Value Calling Convention Implementation 826//===----------------------------------------------------------------------===// 827 828#include "X86GenCallingConv.inc" 829 830/// LowerRET - Lower an ISD::RET node. 831SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) { 832 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 833 834 SmallVector<CCValAssign, 16> RVLocs; 835 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 836 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 837 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 838 CCInfo.AnalyzeReturn(Op.Val, RetCC_X86); 839 840 // If this is the first return lowered for this function, add the regs to the 841 // liveout set for the function. 842 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 843 for (unsigned i = 0; i != RVLocs.size(); ++i) 844 if (RVLocs[i].isRegLoc()) 845 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 846 } 847 SDOperand Chain = Op.getOperand(0); 848 849 // Handle tail call return. 850 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 851 if (Chain.getOpcode() == X86ISD::TAILCALL) { 852 SDOperand TailCall = Chain; 853 SDOperand TargetAddress = TailCall.getOperand(1); 854 SDOperand StackAdjustment = TailCall.getOperand(2); 855 assert(((TargetAddress.getOpcode() == ISD::Register && 856 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 857 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 858 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 859 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 860 "Expecting an global address, external symbol, or register"); 861 assert(StackAdjustment.getOpcode() == ISD::Constant && 862 "Expecting a const value"); 863 864 SmallVector<SDOperand,8> Operands; 865 Operands.push_back(Chain.getOperand(0)); 866 Operands.push_back(TargetAddress); 867 Operands.push_back(StackAdjustment); 868 // Copy registers used by the call. Last operand is a flag so it is not 869 // copied. 870 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 871 Operands.push_back(Chain.getOperand(i)); 872 } 873 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 874 Operands.size()); 875 } 876 877 // Regular return. 878 SDOperand Flag; 879 880 SmallVector<SDOperand, 6> RetOps; 881 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 882 // Operand #1 = Bytes To Pop 883 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 884 885 // Copy the result values into the output registers. 886 for (unsigned i = 0; i != RVLocs.size(); ++i) { 887 CCValAssign &VA = RVLocs[i]; 888 assert(VA.isRegLoc() && "Can only return in registers!"); 889 SDOperand ValToCopy = Op.getOperand(i*2+1); 890 891 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 892 // the RET instruction and handled by the FP Stackifier. 893 if (RVLocs[i].getLocReg() == X86::ST0 || 894 RVLocs[i].getLocReg() == X86::ST1) { 895 // If this is a copy from an xmm register to ST(0), use an FPExtend to 896 // change the value to the FP stack register class. 897 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 898 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 899 RetOps.push_back(ValToCopy); 900 // Don't emit a copytoreg. 901 continue; 902 } 903 904 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 905 Flag = Chain.getValue(1); 906 } 907 908 // The x86-64 ABI for returning structs by value requires that we copy 909 // the sret argument into %rax for the return. We saved the argument into 910 // a virtual register in the entry block, so now we copy the value out 911 // and into %rax. 912 if (Subtarget->is64Bit() && 913 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 914 MachineFunction &MF = DAG.getMachineFunction(); 915 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 916 unsigned Reg = FuncInfo->getSRetReturnReg(); 917 if (!Reg) { 918 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 919 FuncInfo->setSRetReturnReg(Reg); 920 } 921 SDOperand Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 922 923 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 924 Flag = Chain.getValue(1); 925 } 926 927 RetOps[0] = Chain; // Update chain. 928 929 // Add the flag if we have it. 930 if (Flag.Val) 931 RetOps.push_back(Flag); 932 933 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 934} 935 936 937/// LowerCallResult - Lower the result values of an ISD::CALL into the 938/// appropriate copies out of appropriate physical registers. This assumes that 939/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 940/// being lowered. The returns a SDNode with the same number of values as the 941/// ISD::CALL. 942SDNode *X86TargetLowering:: 943LowerCallResult(SDOperand Chain, SDOperand InFlag, SDNode *TheCall, 944 unsigned CallingConv, SelectionDAG &DAG) { 945 946 // Assign locations to each value returned by this call. 947 SmallVector<CCValAssign, 16> RVLocs; 948 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 949 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 950 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 951 952 SmallVector<SDOperand, 8> ResultVals; 953 954 // Copy all of the result registers out of their specified physreg. 955 for (unsigned i = 0; i != RVLocs.size(); ++i) { 956 MVT CopyVT = RVLocs[i].getValVT(); 957 958 // If this is a call to a function that returns an fp value on the floating 959 // point stack, but where we prefer to use the value in xmm registers, copy 960 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 961 if (RVLocs[i].getLocReg() == X86::ST0 && 962 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 963 CopyVT = MVT::f80; 964 } 965 966 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 967 CopyVT, InFlag).getValue(1); 968 SDOperand Val = Chain.getValue(0); 969 InFlag = Chain.getValue(2); 970 971 if (CopyVT != RVLocs[i].getValVT()) { 972 // Round the F80 the right size, which also moves to the appropriate xmm 973 // register. 974 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 975 // This truncation won't change the value. 976 DAG.getIntPtrConstant(1)); 977 } 978 979 ResultVals.push_back(Val); 980 } 981 982 // Merge everything together with a MERGE_VALUES node. 983 ResultVals.push_back(Chain); 984 return DAG.getMergeValues(TheCall->getVTList(), &ResultVals[0], 985 ResultVals.size()).Val; 986} 987 988 989//===----------------------------------------------------------------------===// 990// C & StdCall & Fast Calling Convention implementation 991//===----------------------------------------------------------------------===// 992// StdCall calling convention seems to be standard for many Windows' API 993// routines and around. It differs from C calling convention just a little: 994// callee should clean up the stack, not caller. Symbols should be also 995// decorated in some fancy way :) It doesn't support any vector arguments. 996// For info on fast calling convention see Fast Calling Convention (tail call) 997// implementation LowerX86_32FastCCCallTo. 998 999/// AddLiveIn - This helper function adds the specified physical register to the 1000/// MachineFunction as a live in value. It also creates a corresponding virtual 1001/// register for it. 1002static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1003 const TargetRegisterClass *RC) { 1004 assert(RC->contains(PReg) && "Not the correct regclass!"); 1005 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1006 MF.getRegInfo().addLiveIn(PReg, VReg); 1007 return VReg; 1008} 1009 1010/// CallIsStructReturn - Determines whether a CALL node uses struct return 1011/// semantics. 1012static bool CallIsStructReturn(SDOperand Op) { 1013 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1014 if (!NumOps) 1015 return false; 1016 1017 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1018} 1019 1020/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1021/// return semantics. 1022static bool ArgsAreStructReturn(SDOperand Op) { 1023 unsigned NumArgs = Op.Val->getNumValues() - 1; 1024 if (!NumArgs) 1025 return false; 1026 1027 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1028} 1029 1030/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1031/// the callee to pop its own arguments. Callee pop is necessary to support tail 1032/// calls. 1033bool X86TargetLowering::IsCalleePop(SDOperand Op) { 1034 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1035 if (IsVarArg) 1036 return false; 1037 1038 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1039 default: 1040 return false; 1041 case CallingConv::X86_StdCall: 1042 return !Subtarget->is64Bit(); 1043 case CallingConv::X86_FastCall: 1044 return !Subtarget->is64Bit(); 1045 case CallingConv::Fast: 1046 return PerformTailCallOpt; 1047 } 1048} 1049 1050/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1051/// FORMAL_ARGUMENTS node. 1052CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDOperand Op) const { 1053 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1054 1055 if (Subtarget->is64Bit()) { 1056 if (Subtarget->isTargetWin64()) 1057 return CC_X86_Win64_C; 1058 else { 1059 if (CC == CallingConv::Fast && PerformTailCallOpt) 1060 return CC_X86_64_TailCall; 1061 else 1062 return CC_X86_64_C; 1063 } 1064 } 1065 1066 if (CC == CallingConv::X86_FastCall) 1067 return CC_X86_32_FastCall; 1068 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1069 return CC_X86_32_TailCall; 1070 else 1071 return CC_X86_32_C; 1072} 1073 1074/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1075/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1076NameDecorationStyle 1077X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDOperand Op) { 1078 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1079 if (CC == CallingConv::X86_FastCall) 1080 return FastCall; 1081 else if (CC == CallingConv::X86_StdCall) 1082 return StdCall; 1083 return None; 1084} 1085 1086 1087/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1088/// in a register before calling. 1089bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1090 return !IsTailCall && !Is64Bit && 1091 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1092 Subtarget->isPICStyleGOT(); 1093} 1094 1095/// CallRequiresFnAddressInReg - Check whether the call requires the function 1096/// address to be loaded in a register. 1097bool 1098X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1099 return !Is64Bit && IsTailCall && 1100 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1101 Subtarget->isPICStyleGOT(); 1102} 1103 1104/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1105/// by "Src" to address "Dst" with size and alignment information specified by 1106/// the specific parameter attribute. The copy will be passed as a byval 1107/// function parameter. 1108static SDOperand 1109CreateCopyOfByValArgument(SDOperand Src, SDOperand Dst, SDOperand Chain, 1110 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1111 SDOperand SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1112 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1113 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1114} 1115 1116SDOperand X86TargetLowering::LowerMemArgument(SDOperand Op, SelectionDAG &DAG, 1117 const CCValAssign &VA, 1118 MachineFrameInfo *MFI, 1119 unsigned CC, 1120 SDOperand Root, unsigned i) { 1121 // Create the nodes corresponding to a load from this parameter slot. 1122 ISD::ArgFlagsTy Flags = 1123 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1124 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1125 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1126 1127 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1128 // changed with more analysis. 1129 // In case of tail call optimization mark all arguments mutable. Since they 1130 // could be overwritten by lowering of arguments in case of a tail call. 1131 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1132 VA.getLocMemOffset(), isImmutable); 1133 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); 1134 if (Flags.isByVal()) 1135 return FIN; 1136 return DAG.getLoad(VA.getValVT(), Root, FIN, 1137 PseudoSourceValue::getFixedStack(FI), 0); 1138} 1139 1140SDOperand 1141X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) { 1142 MachineFunction &MF = DAG.getMachineFunction(); 1143 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1144 1145 const Function* Fn = MF.getFunction(); 1146 if (Fn->hasExternalLinkage() && 1147 Subtarget->isTargetCygMing() && 1148 Fn->getName() == "main") 1149 FuncInfo->setForceFramePointer(true); 1150 1151 // Decorate the function name. 1152 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1153 1154 MachineFrameInfo *MFI = MF.getFrameInfo(); 1155 SDOperand Root = Op.getOperand(0); 1156 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1157 unsigned CC = MF.getFunction()->getCallingConv(); 1158 bool Is64Bit = Subtarget->is64Bit(); 1159 bool IsWin64 = Subtarget->isTargetWin64(); 1160 1161 assert(!(isVarArg && CC == CallingConv::Fast) && 1162 "Var args not supported with calling convention fastcc"); 1163 1164 // Assign locations to all of the incoming arguments. 1165 SmallVector<CCValAssign, 16> ArgLocs; 1166 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1167 CCInfo.AnalyzeFormalArguments(Op.Val, CCAssignFnForNode(Op)); 1168 1169 SmallVector<SDOperand, 8> ArgValues; 1170 unsigned LastVal = ~0U; 1171 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1172 CCValAssign &VA = ArgLocs[i]; 1173 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1174 // places. 1175 assert(VA.getValNo() != LastVal && 1176 "Don't support value assigned to multiple locs yet"); 1177 LastVal = VA.getValNo(); 1178 1179 if (VA.isRegLoc()) { 1180 MVT RegVT = VA.getLocVT(); 1181 TargetRegisterClass *RC; 1182 if (RegVT == MVT::i32) 1183 RC = X86::GR32RegisterClass; 1184 else if (Is64Bit && RegVT == MVT::i64) 1185 RC = X86::GR64RegisterClass; 1186 else if (RegVT == MVT::f32) 1187 RC = X86::FR32RegisterClass; 1188 else if (RegVT == MVT::f64) 1189 RC = X86::FR64RegisterClass; 1190 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1191 RC = X86::VR128RegisterClass; 1192 else if (RegVT.isVector()) { 1193 assert(RegVT.getSizeInBits() == 64); 1194 if (!Is64Bit) 1195 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1196 else { 1197 // Darwin calling convention passes MMX values in either GPRs or 1198 // XMMs in x86-64. Other targets pass them in memory. 1199 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1200 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1201 RegVT = MVT::v2i64; 1202 } else { 1203 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1204 RegVT = MVT::i64; 1205 } 1206 } 1207 } else { 1208 assert(0 && "Unknown argument type!"); 1209 } 1210 1211 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1212 SDOperand ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1213 1214 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1215 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1216 // right size. 1217 if (VA.getLocInfo() == CCValAssign::SExt) 1218 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1219 DAG.getValueType(VA.getValVT())); 1220 else if (VA.getLocInfo() == CCValAssign::ZExt) 1221 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1222 DAG.getValueType(VA.getValVT())); 1223 1224 if (VA.getLocInfo() != CCValAssign::Full) 1225 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1226 1227 // Handle MMX values passed in GPRs. 1228 if (Is64Bit && RegVT != VA.getLocVT()) { 1229 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1230 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1231 else if (RC == X86::VR128RegisterClass) { 1232 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1233 DAG.getConstant(0, MVT::i64)); 1234 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1235 } 1236 } 1237 1238 ArgValues.push_back(ArgValue); 1239 } else { 1240 assert(VA.isMemLoc()); 1241 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1242 } 1243 } 1244 1245 // The x86-64 ABI for returning structs by value requires that we copy 1246 // the sret argument into %rax for the return. Save the argument into 1247 // a virtual register so that we can access it from the return points. 1248 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1249 MachineFunction &MF = DAG.getMachineFunction(); 1250 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1251 unsigned Reg = FuncInfo->getSRetReturnReg(); 1252 if (!Reg) { 1253 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1254 FuncInfo->setSRetReturnReg(Reg); 1255 } 1256 SDOperand Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1257 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1258 } 1259 1260 unsigned StackSize = CCInfo.getNextStackOffset(); 1261 // align stack specially for tail calls 1262 if (CC == CallingConv::Fast) 1263 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1264 1265 // If the function takes variable number of arguments, make a frame index for 1266 // the start of the first vararg value... for expansion of llvm.va_start. 1267 if (isVarArg) { 1268 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1269 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1270 } 1271 if (Is64Bit) { 1272 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1273 1274 // FIXME: We should really autogenerate these arrays 1275 static const unsigned GPR64ArgRegsWin64[] = { 1276 X86::RCX, X86::RDX, X86::R8, X86::R9 1277 }; 1278 static const unsigned XMMArgRegsWin64[] = { 1279 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1280 }; 1281 static const unsigned GPR64ArgRegs64Bit[] = { 1282 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1283 }; 1284 static const unsigned XMMArgRegs64Bit[] = { 1285 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1286 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1287 }; 1288 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1289 1290 if (IsWin64) { 1291 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1292 GPR64ArgRegs = GPR64ArgRegsWin64; 1293 XMMArgRegs = XMMArgRegsWin64; 1294 } else { 1295 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1296 GPR64ArgRegs = GPR64ArgRegs64Bit; 1297 XMMArgRegs = XMMArgRegs64Bit; 1298 } 1299 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1300 TotalNumIntRegs); 1301 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1302 TotalNumXMMRegs); 1303 1304 // For X86-64, if there are vararg parameters that are passed via 1305 // registers, then we must store them to their spots on the stack so they 1306 // may be loaded by deferencing the result of va_next. 1307 VarArgsGPOffset = NumIntRegs * 8; 1308 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1309 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1310 TotalNumXMMRegs * 16, 16); 1311 1312 // Store the integer parameter registers. 1313 SmallVector<SDOperand, 8> MemOps; 1314 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1315 SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1316 DAG.getIntPtrConstant(VarArgsGPOffset)); 1317 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1318 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1319 X86::GR64RegisterClass); 1320 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1321 SDOperand Store = 1322 DAG.getStore(Val.getValue(1), Val, FIN, 1323 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1324 MemOps.push_back(Store); 1325 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1326 DAG.getIntPtrConstant(8)); 1327 } 1328 1329 // Now store the XMM (fp + vector) parameter registers. 1330 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1331 DAG.getIntPtrConstant(VarArgsFPOffset)); 1332 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1333 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1334 X86::VR128RegisterClass); 1335 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1336 SDOperand Store = 1337 DAG.getStore(Val.getValue(1), Val, FIN, 1338 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1339 MemOps.push_back(Store); 1340 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1341 DAG.getIntPtrConstant(16)); 1342 } 1343 if (!MemOps.empty()) 1344 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1345 &MemOps[0], MemOps.size()); 1346 } 1347 } 1348 1349 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1350 // arguments and the arguments after the retaddr has been pushed are 1351 // aligned. 1352 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1353 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1354 (StackSize & 7) == 0) 1355 StackSize += 4; 1356 1357 ArgValues.push_back(Root); 1358 1359 // Some CCs need callee pop. 1360 if (IsCalleePop(Op)) { 1361 BytesToPopOnReturn = StackSize; // Callee pops everything. 1362 BytesCallerReserves = 0; 1363 } else { 1364 BytesToPopOnReturn = 0; // Callee pops nothing. 1365 // If this is an sret function, the return should pop the hidden pointer. 1366 if (!Is64Bit && ArgsAreStructReturn(Op)) 1367 BytesToPopOnReturn = 4; 1368 BytesCallerReserves = StackSize; 1369 } 1370 1371 if (!Is64Bit) { 1372 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1373 if (CC == CallingConv::X86_FastCall) 1374 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1375 } 1376 1377 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1378 1379 // Return the new list of results. 1380 return DAG.getMergeValues(Op.Val->getVTList(), &ArgValues[0], 1381 ArgValues.size()).getValue(Op.ResNo); 1382} 1383 1384SDOperand 1385X86TargetLowering::LowerMemOpCallTo(SDOperand Op, SelectionDAG &DAG, 1386 const SDOperand &StackPtr, 1387 const CCValAssign &VA, 1388 SDOperand Chain, 1389 SDOperand Arg) { 1390 unsigned LocMemOffset = VA.getLocMemOffset(); 1391 SDOperand PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1392 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1393 ISD::ArgFlagsTy Flags = 1394 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1395 if (Flags.isByVal()) { 1396 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1397 } 1398 return DAG.getStore(Chain, Arg, PtrOff, 1399 PseudoSourceValue::getStack(), LocMemOffset); 1400} 1401 1402/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1403/// optimization is performed and it is required. 1404SDOperand 1405X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1406 SDOperand &OutRetAddr, 1407 SDOperand Chain, 1408 bool IsTailCall, 1409 bool Is64Bit, 1410 int FPDiff) { 1411 if (!IsTailCall || FPDiff==0) return Chain; 1412 1413 // Adjust the Return address stack slot. 1414 MVT VT = getPointerTy(); 1415 OutRetAddr = getReturnAddressFrameIndex(DAG); 1416 // Load the "old" Return address. 1417 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1418 return SDOperand(OutRetAddr.Val, 1); 1419} 1420 1421/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1422/// optimization is performed and it is required (FPDiff!=0). 1423static SDOperand 1424EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1425 SDOperand Chain, SDOperand RetAddrFrIdx, 1426 bool Is64Bit, int FPDiff) { 1427 // Store the return address to the appropriate stack slot. 1428 if (!FPDiff) return Chain; 1429 // Calculate the new stack slot for the return address. 1430 int SlotSize = Is64Bit ? 8 : 4; 1431 int NewReturnAddrFI = 1432 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1433 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1434 SDOperand NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1435 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1436 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 1437 return Chain; 1438} 1439 1440SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) { 1441 MachineFunction &MF = DAG.getMachineFunction(); 1442 SDOperand Chain = Op.getOperand(0); 1443 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1444 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1445 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1446 && CC == CallingConv::Fast && PerformTailCallOpt; 1447 SDOperand Callee = Op.getOperand(4); 1448 bool Is64Bit = Subtarget->is64Bit(); 1449 bool IsStructRet = CallIsStructReturn(Op); 1450 1451 assert(!(isVarArg && CC == CallingConv::Fast) && 1452 "Var args not supported with calling convention fastcc"); 1453 1454 // Analyze operands of the call, assigning locations to each operand. 1455 SmallVector<CCValAssign, 16> ArgLocs; 1456 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1457 CCInfo.AnalyzeCallOperands(Op.Val, CCAssignFnForNode(Op)); 1458 1459 // Get a count of how many bytes are to be pushed on the stack. 1460 unsigned NumBytes = CCInfo.getNextStackOffset(); 1461 if (CC == CallingConv::Fast) 1462 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1463 1464 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1465 // arguments and the arguments after the retaddr has been pushed are aligned. 1466 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1467 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1468 (NumBytes & 7) == 0) 1469 NumBytes += 4; 1470 1471 int FPDiff = 0; 1472 if (IsTailCall) { 1473 // Lower arguments at fp - stackoffset + fpdiff. 1474 unsigned NumBytesCallerPushed = 1475 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1476 FPDiff = NumBytesCallerPushed - NumBytes; 1477 1478 // Set the delta of movement of the returnaddr stackslot. 1479 // But only set if delta is greater than previous delta. 1480 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1481 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1482 } 1483 1484 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1485 1486 SDOperand RetAddrFrIdx; 1487 // Load return adress for tail calls. 1488 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1489 FPDiff); 1490 1491 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass; 1492 SmallVector<SDOperand, 8> MemOpChains; 1493 SDOperand StackPtr; 1494 1495 // Walk the register/memloc assignments, inserting copies/loads. In the case 1496 // of tail call optimization arguments are handle later. 1497 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1498 CCValAssign &VA = ArgLocs[i]; 1499 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1500 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1501 getArgFlags().isByVal(); 1502 1503 // Promote the value if needed. 1504 switch (VA.getLocInfo()) { 1505 default: assert(0 && "Unknown loc info!"); 1506 case CCValAssign::Full: break; 1507 case CCValAssign::SExt: 1508 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1509 break; 1510 case CCValAssign::ZExt: 1511 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1512 break; 1513 case CCValAssign::AExt: 1514 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1515 break; 1516 } 1517 1518 if (VA.isRegLoc()) { 1519 if (Is64Bit) { 1520 MVT RegVT = VA.getLocVT(); 1521 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1522 switch (VA.getLocReg()) { 1523 default: 1524 break; 1525 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1526 case X86::R8: { 1527 // Special case: passing MMX values in GPR registers. 1528 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1529 break; 1530 } 1531 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1532 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1533 // Special case: passing MMX values in XMM registers. 1534 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1535 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1536 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1537 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1538 getMOVLMask(2, DAG)); 1539 break; 1540 } 1541 } 1542 } 1543 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1544 } else { 1545 if (!IsTailCall || (IsTailCall && isByVal)) { 1546 assert(VA.isMemLoc()); 1547 if (StackPtr.Val == 0) 1548 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1549 1550 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1551 Arg)); 1552 } 1553 } 1554 } 1555 1556 if (!MemOpChains.empty()) 1557 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1558 &MemOpChains[0], MemOpChains.size()); 1559 1560 // Build a sequence of copy-to-reg nodes chained together with token chain 1561 // and flag operands which copy the outgoing args into registers. 1562 SDOperand InFlag; 1563 // Tail call byval lowering might overwrite argument registers so in case of 1564 // tail call optimization the copies to registers are lowered later. 1565 if (!IsTailCall) 1566 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1567 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1568 InFlag); 1569 InFlag = Chain.getValue(1); 1570 } 1571 1572 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1573 // GOT pointer. 1574 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1575 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1576 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1577 InFlag); 1578 InFlag = Chain.getValue(1); 1579 } 1580 // If we are tail calling and generating PIC/GOT style code load the address 1581 // of the callee into ecx. The value in ecx is used as target of the tail 1582 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1583 // calls on PIC/GOT architectures. Normally we would just put the address of 1584 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1585 // restored (since ebx is callee saved) before jumping to the target@PLT. 1586 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1587 // Note: The actual moving to ecx is done further down. 1588 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1589 if (G && !G->getGlobal()->hasHiddenVisibility() && 1590 !G->getGlobal()->hasProtectedVisibility()) 1591 Callee = LowerGlobalAddress(Callee, DAG); 1592 else if (isa<ExternalSymbolSDNode>(Callee)) 1593 Callee = LowerExternalSymbol(Callee,DAG); 1594 } 1595 1596 if (Is64Bit && isVarArg) { 1597 // From AMD64 ABI document: 1598 // For calls that may call functions that use varargs or stdargs 1599 // (prototype-less calls or calls to functions containing ellipsis (...) in 1600 // the declaration) %al is used as hidden argument to specify the number 1601 // of SSE registers used. The contents of %al do not need to match exactly 1602 // the number of registers, but must be an ubound on the number of SSE 1603 // registers used and is in the range 0 - 8 inclusive. 1604 1605 // FIXME: Verify this on Win64 1606 // Count the number of XMM registers allocated. 1607 static const unsigned XMMArgRegs[] = { 1608 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1609 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1610 }; 1611 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1612 1613 Chain = DAG.getCopyToReg(Chain, X86::AL, 1614 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1615 InFlag = Chain.getValue(1); 1616 } 1617 1618 1619 // For tail calls lower the arguments to the 'real' stack slot. 1620 if (IsTailCall) { 1621 SmallVector<SDOperand, 8> MemOpChains2; 1622 SDOperand FIN; 1623 int FI = 0; 1624 // Do not flag preceeding copytoreg stuff together with the following stuff. 1625 InFlag = SDOperand(); 1626 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1627 CCValAssign &VA = ArgLocs[i]; 1628 if (!VA.isRegLoc()) { 1629 assert(VA.isMemLoc()); 1630 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1631 SDOperand FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1632 ISD::ArgFlagsTy Flags = 1633 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1634 // Create frame index. 1635 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1636 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1637 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1638 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1639 1640 if (Flags.isByVal()) { 1641 // Copy relative to framepointer. 1642 SDOperand Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1643 if (StackPtr.Val == 0) 1644 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1645 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1646 1647 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1648 Flags, DAG)); 1649 } else { 1650 // Store relative to framepointer. 1651 MemOpChains2.push_back( 1652 DAG.getStore(Chain, Arg, FIN, 1653 PseudoSourceValue::getFixedStack(FI), 0)); 1654 } 1655 } 1656 } 1657 1658 if (!MemOpChains2.empty()) 1659 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1660 &MemOpChains2[0], MemOpChains2.size()); 1661 1662 // Copy arguments to their registers. 1663 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1664 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1665 InFlag); 1666 InFlag = Chain.getValue(1); 1667 } 1668 InFlag =SDOperand(); 1669 1670 // Store the return address to the appropriate stack slot. 1671 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1672 FPDiff); 1673 } 1674 1675 // If the callee is a GlobalAddress node (quite common, every direct call is) 1676 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1677 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1678 // We should use extra load for direct calls to dllimported functions in 1679 // non-JIT mode. 1680 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1681 getTargetMachine(), true)) 1682 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1683 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1684 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1685 } else if (IsTailCall) { 1686 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1687 1688 Chain = DAG.getCopyToReg(Chain, 1689 DAG.getRegister(Opc, getPointerTy()), 1690 Callee,InFlag); 1691 Callee = DAG.getRegister(Opc, getPointerTy()); 1692 // Add register as live out. 1693 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1694 } 1695 1696 // Returns a chain & a flag for retval copy to use. 1697 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1698 SmallVector<SDOperand, 8> Ops; 1699 1700 if (IsTailCall) { 1701 Ops.push_back(Chain); 1702 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1703 Ops.push_back(DAG.getIntPtrConstant(0)); 1704 if (InFlag.Val) 1705 Ops.push_back(InFlag); 1706 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1707 InFlag = Chain.getValue(1); 1708 1709 // Returns a chain & a flag for retval copy to use. 1710 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1711 Ops.clear(); 1712 } 1713 1714 Ops.push_back(Chain); 1715 Ops.push_back(Callee); 1716 1717 if (IsTailCall) 1718 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1719 1720 // Add argument registers to the end of the list so that they are known live 1721 // into the call. 1722 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1723 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1724 RegsToPass[i].second.getValueType())); 1725 1726 // Add an implicit use GOT pointer in EBX. 1727 if (!IsTailCall && !Is64Bit && 1728 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1729 Subtarget->isPICStyleGOT()) 1730 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1731 1732 // Add an implicit use of AL for x86 vararg functions. 1733 if (Is64Bit && isVarArg) 1734 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1735 1736 if (InFlag.Val) 1737 Ops.push_back(InFlag); 1738 1739 if (IsTailCall) { 1740 assert(InFlag.Val && 1741 "Flag must be set. Depend on flag being set in LowerRET"); 1742 Chain = DAG.getNode(X86ISD::TAILCALL, 1743 Op.Val->getVTList(), &Ops[0], Ops.size()); 1744 1745 return SDOperand(Chain.Val, Op.ResNo); 1746 } 1747 1748 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1749 InFlag = Chain.getValue(1); 1750 1751 // Create the CALLSEQ_END node. 1752 unsigned NumBytesForCalleeToPush; 1753 if (IsCalleePop(Op)) 1754 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1755 else if (!Is64Bit && IsStructRet) 1756 // If this is is a call to a struct-return function, the callee 1757 // pops the hidden struct pointer, so we have to push it back. 1758 // This is common for Darwin/X86, Linux & Mingw32 targets. 1759 NumBytesForCalleeToPush = 4; 1760 else 1761 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1762 1763 // Returns a flag for retval copy to use. 1764 Chain = DAG.getCALLSEQ_END(Chain, 1765 DAG.getIntPtrConstant(NumBytes), 1766 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1767 InFlag); 1768 InFlag = Chain.getValue(1); 1769 1770 // Handle result values, copying them out of physregs into vregs that we 1771 // return. 1772 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo); 1773} 1774 1775 1776//===----------------------------------------------------------------------===// 1777// Fast Calling Convention (tail call) implementation 1778//===----------------------------------------------------------------------===// 1779 1780// Like std call, callee cleans arguments, convention except that ECX is 1781// reserved for storing the tail called function address. Only 2 registers are 1782// free for argument passing (inreg). Tail call optimization is performed 1783// provided: 1784// * tailcallopt is enabled 1785// * caller/callee are fastcc 1786// On X86_64 architecture with GOT-style position independent code only local 1787// (within module) calls are supported at the moment. 1788// To keep the stack aligned according to platform abi the function 1789// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1790// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1791// If a tail called function callee has more arguments than the caller the 1792// caller needs to make sure that there is room to move the RETADDR to. This is 1793// achieved by reserving an area the size of the argument delta right after the 1794// original REtADDR, but before the saved framepointer or the spilled registers 1795// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1796// stack layout: 1797// arg1 1798// arg2 1799// RETADDR 1800// [ new RETADDR 1801// move area ] 1802// (possible EBP) 1803// ESI 1804// EDI 1805// local1 .. 1806 1807/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1808/// for a 16 byte align requirement. 1809unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1810 SelectionDAG& DAG) { 1811 if (PerformTailCallOpt) { 1812 MachineFunction &MF = DAG.getMachineFunction(); 1813 const TargetMachine &TM = MF.getTarget(); 1814 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1815 unsigned StackAlignment = TFI.getStackAlignment(); 1816 uint64_t AlignMask = StackAlignment - 1; 1817 int64_t Offset = StackSize; 1818 unsigned SlotSize = Subtarget->is64Bit() ? 8 : 4; 1819 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1820 // Number smaller than 12 so just add the difference. 1821 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1822 } else { 1823 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1824 Offset = ((~AlignMask) & Offset) + StackAlignment + 1825 (StackAlignment-SlotSize); 1826 } 1827 StackSize = Offset; 1828 } 1829 return StackSize; 1830} 1831 1832/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1833/// following the call is a return. A function is eligible if caller/callee 1834/// calling conventions match, currently only fastcc supports tail calls, and 1835/// the function CALL is immediatly followed by a RET. 1836bool X86TargetLowering::IsEligibleForTailCallOptimization(SDOperand Call, 1837 SDOperand Ret, 1838 SelectionDAG& DAG) const { 1839 if (!PerformTailCallOpt) 1840 return false; 1841 1842 if (CheckTailCallReturnConstraints(Call, Ret)) { 1843 MachineFunction &MF = DAG.getMachineFunction(); 1844 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1845 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1846 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1847 SDOperand Callee = Call.getOperand(4); 1848 // On x86/32Bit PIC/GOT tail calls are supported. 1849 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1850 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1851 return true; 1852 1853 // Can only do local tail calls (in same module, hidden or protected) on 1854 // x86_64 PIC/GOT at the moment. 1855 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1856 return G->getGlobal()->hasHiddenVisibility() 1857 || G->getGlobal()->hasProtectedVisibility(); 1858 } 1859 } 1860 1861 return false; 1862} 1863 1864//===----------------------------------------------------------------------===// 1865// Other Lowering Hooks 1866//===----------------------------------------------------------------------===// 1867 1868 1869SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1870 MachineFunction &MF = DAG.getMachineFunction(); 1871 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1872 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1873 1874 if (ReturnAddrIndex == 0) { 1875 // Set up a frame object for the return address. 1876 if (Subtarget->is64Bit()) 1877 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); 1878 else 1879 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1880 1881 FuncInfo->setRAIndex(ReturnAddrIndex); 1882 } 1883 1884 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1885} 1886 1887 1888 1889/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1890/// specific condition code. It returns a false if it cannot do a direct 1891/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1892/// needed. 1893static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1894 unsigned &X86CC, SDOperand &LHS, SDOperand &RHS, 1895 SelectionDAG &DAG) { 1896 X86CC = X86::COND_INVALID; 1897 if (!isFP) { 1898 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1899 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1900 // X > -1 -> X == 0, jump !sign. 1901 RHS = DAG.getConstant(0, RHS.getValueType()); 1902 X86CC = X86::COND_NS; 1903 return true; 1904 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1905 // X < 0 -> X == 0, jump on sign. 1906 X86CC = X86::COND_S; 1907 return true; 1908 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1909 // X < 1 -> X <= 0 1910 RHS = DAG.getConstant(0, RHS.getValueType()); 1911 X86CC = X86::COND_LE; 1912 return true; 1913 } 1914 } 1915 1916 switch (SetCCOpcode) { 1917 default: break; 1918 case ISD::SETEQ: X86CC = X86::COND_E; break; 1919 case ISD::SETGT: X86CC = X86::COND_G; break; 1920 case ISD::SETGE: X86CC = X86::COND_GE; break; 1921 case ISD::SETLT: X86CC = X86::COND_L; break; 1922 case ISD::SETLE: X86CC = X86::COND_LE; break; 1923 case ISD::SETNE: X86CC = X86::COND_NE; break; 1924 case ISD::SETULT: X86CC = X86::COND_B; break; 1925 case ISD::SETUGT: X86CC = X86::COND_A; break; 1926 case ISD::SETULE: X86CC = X86::COND_BE; break; 1927 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1928 } 1929 } else { 1930 // On a floating point condition, the flags are set as follows: 1931 // ZF PF CF op 1932 // 0 | 0 | 0 | X > Y 1933 // 0 | 0 | 1 | X < Y 1934 // 1 | 0 | 0 | X == Y 1935 // 1 | 1 | 1 | unordered 1936 bool Flip = false; 1937 switch (SetCCOpcode) { 1938 default: break; 1939 case ISD::SETUEQ: 1940 case ISD::SETEQ: X86CC = X86::COND_E; break; 1941 case ISD::SETOLT: Flip = true; // Fallthrough 1942 case ISD::SETOGT: 1943 case ISD::SETGT: X86CC = X86::COND_A; break; 1944 case ISD::SETOLE: Flip = true; // Fallthrough 1945 case ISD::SETOGE: 1946 case ISD::SETGE: X86CC = X86::COND_AE; break; 1947 case ISD::SETUGT: Flip = true; // Fallthrough 1948 case ISD::SETULT: 1949 case ISD::SETLT: X86CC = X86::COND_B; break; 1950 case ISD::SETUGE: Flip = true; // Fallthrough 1951 case ISD::SETULE: 1952 case ISD::SETLE: X86CC = X86::COND_BE; break; 1953 case ISD::SETONE: 1954 case ISD::SETNE: X86CC = X86::COND_NE; break; 1955 case ISD::SETUO: X86CC = X86::COND_P; break; 1956 case ISD::SETO: X86CC = X86::COND_NP; break; 1957 } 1958 if (Flip) 1959 std::swap(LHS, RHS); 1960 } 1961 1962 return X86CC != X86::COND_INVALID; 1963} 1964 1965/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1966/// code. Current x86 isa includes the following FP cmov instructions: 1967/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1968static bool hasFPCMov(unsigned X86CC) { 1969 switch (X86CC) { 1970 default: 1971 return false; 1972 case X86::COND_B: 1973 case X86::COND_BE: 1974 case X86::COND_E: 1975 case X86::COND_P: 1976 case X86::COND_A: 1977 case X86::COND_AE: 1978 case X86::COND_NE: 1979 case X86::COND_NP: 1980 return true; 1981 } 1982} 1983 1984/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 1985/// true if Op is undef or if its value falls within the specified range (L, H]. 1986static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) { 1987 if (Op.getOpcode() == ISD::UNDEF) 1988 return true; 1989 1990 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 1991 return (Val >= Low && Val < Hi); 1992} 1993 1994/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 1995/// true if Op is undef or if its value equal to the specified value. 1996static bool isUndefOrEqual(SDOperand Op, unsigned Val) { 1997 if (Op.getOpcode() == ISD::UNDEF) 1998 return true; 1999 return cast<ConstantSDNode>(Op)->getValue() == Val; 2000} 2001 2002/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2003/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2004bool X86::isPSHUFDMask(SDNode *N) { 2005 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2006 2007 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2008 return false; 2009 2010 // Check if the value doesn't reference the second vector. 2011 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2012 SDOperand Arg = N->getOperand(i); 2013 if (Arg.getOpcode() == ISD::UNDEF) continue; 2014 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2015 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2016 return false; 2017 } 2018 2019 return true; 2020} 2021 2022/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2023/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2024bool X86::isPSHUFHWMask(SDNode *N) { 2025 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2026 2027 if (N->getNumOperands() != 8) 2028 return false; 2029 2030 // Lower quadword copied in order. 2031 for (unsigned i = 0; i != 4; ++i) { 2032 SDOperand Arg = N->getOperand(i); 2033 if (Arg.getOpcode() == ISD::UNDEF) continue; 2034 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2035 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2036 return false; 2037 } 2038 2039 // Upper quadword shuffled. 2040 for (unsigned i = 4; i != 8; ++i) { 2041 SDOperand Arg = N->getOperand(i); 2042 if (Arg.getOpcode() == ISD::UNDEF) continue; 2043 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2044 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2045 if (Val < 4 || Val > 7) 2046 return false; 2047 } 2048 2049 return true; 2050} 2051 2052/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2053/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2054bool X86::isPSHUFLWMask(SDNode *N) { 2055 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2056 2057 if (N->getNumOperands() != 8) 2058 return false; 2059 2060 // Upper quadword copied in order. 2061 for (unsigned i = 4; i != 8; ++i) 2062 if (!isUndefOrEqual(N->getOperand(i), i)) 2063 return false; 2064 2065 // Lower quadword shuffled. 2066 for (unsigned i = 0; i != 4; ++i) 2067 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2068 return false; 2069 2070 return true; 2071} 2072 2073/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2074/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2075static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2076 if (NumElems != 2 && NumElems != 4) return false; 2077 2078 unsigned Half = NumElems / 2; 2079 for (unsigned i = 0; i < Half; ++i) 2080 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2081 return false; 2082 for (unsigned i = Half; i < NumElems; ++i) 2083 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2084 return false; 2085 2086 return true; 2087} 2088 2089bool X86::isSHUFPMask(SDNode *N) { 2090 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2091 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2092} 2093 2094/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2095/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2096/// half elements to come from vector 1 (which would equal the dest.) and 2097/// the upper half to come from vector 2. 2098static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2099 if (NumOps != 2 && NumOps != 4) return false; 2100 2101 unsigned Half = NumOps / 2; 2102 for (unsigned i = 0; i < Half; ++i) 2103 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2104 return false; 2105 for (unsigned i = Half; i < NumOps; ++i) 2106 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2107 return false; 2108 return true; 2109} 2110 2111static bool isCommutedSHUFP(SDNode *N) { 2112 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2113 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2114} 2115 2116/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2117/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2118bool X86::isMOVHLPSMask(SDNode *N) { 2119 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2120 2121 if (N->getNumOperands() != 4) 2122 return false; 2123 2124 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2125 return isUndefOrEqual(N->getOperand(0), 6) && 2126 isUndefOrEqual(N->getOperand(1), 7) && 2127 isUndefOrEqual(N->getOperand(2), 2) && 2128 isUndefOrEqual(N->getOperand(3), 3); 2129} 2130 2131/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2132/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2133/// <2, 3, 2, 3> 2134bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2135 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2136 2137 if (N->getNumOperands() != 4) 2138 return false; 2139 2140 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2141 return isUndefOrEqual(N->getOperand(0), 2) && 2142 isUndefOrEqual(N->getOperand(1), 3) && 2143 isUndefOrEqual(N->getOperand(2), 2) && 2144 isUndefOrEqual(N->getOperand(3), 3); 2145} 2146 2147/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2148/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2149bool X86::isMOVLPMask(SDNode *N) { 2150 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2151 2152 unsigned NumElems = N->getNumOperands(); 2153 if (NumElems != 2 && NumElems != 4) 2154 return false; 2155 2156 for (unsigned i = 0; i < NumElems/2; ++i) 2157 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2158 return false; 2159 2160 for (unsigned i = NumElems/2; i < NumElems; ++i) 2161 if (!isUndefOrEqual(N->getOperand(i), i)) 2162 return false; 2163 2164 return true; 2165} 2166 2167/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2168/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2169/// and MOVLHPS. 2170bool X86::isMOVHPMask(SDNode *N) { 2171 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2172 2173 unsigned NumElems = N->getNumOperands(); 2174 if (NumElems != 2 && NumElems != 4) 2175 return false; 2176 2177 for (unsigned i = 0; i < NumElems/2; ++i) 2178 if (!isUndefOrEqual(N->getOperand(i), i)) 2179 return false; 2180 2181 for (unsigned i = 0; i < NumElems/2; ++i) { 2182 SDOperand Arg = N->getOperand(i + NumElems/2); 2183 if (!isUndefOrEqual(Arg, i + NumElems)) 2184 return false; 2185 } 2186 2187 return true; 2188} 2189 2190/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2191/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2192bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2193 bool V2IsSplat = false) { 2194 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2195 return false; 2196 2197 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2198 SDOperand BitI = Elts[i]; 2199 SDOperand BitI1 = Elts[i+1]; 2200 if (!isUndefOrEqual(BitI, j)) 2201 return false; 2202 if (V2IsSplat) { 2203 if (isUndefOrEqual(BitI1, NumElts)) 2204 return false; 2205 } else { 2206 if (!isUndefOrEqual(BitI1, j + NumElts)) 2207 return false; 2208 } 2209 } 2210 2211 return true; 2212} 2213 2214bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2215 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2216 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2217} 2218 2219/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2220/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2221bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2222 bool V2IsSplat = false) { 2223 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2224 return false; 2225 2226 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2227 SDOperand BitI = Elts[i]; 2228 SDOperand BitI1 = Elts[i+1]; 2229 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2230 return false; 2231 if (V2IsSplat) { 2232 if (isUndefOrEqual(BitI1, NumElts)) 2233 return false; 2234 } else { 2235 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2236 return false; 2237 } 2238 } 2239 2240 return true; 2241} 2242 2243bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2244 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2245 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2246} 2247 2248/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2249/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2250/// <0, 0, 1, 1> 2251bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2252 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2253 2254 unsigned NumElems = N->getNumOperands(); 2255 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2256 return false; 2257 2258 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2259 SDOperand BitI = N->getOperand(i); 2260 SDOperand BitI1 = N->getOperand(i+1); 2261 2262 if (!isUndefOrEqual(BitI, j)) 2263 return false; 2264 if (!isUndefOrEqual(BitI1, j)) 2265 return false; 2266 } 2267 2268 return true; 2269} 2270 2271/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2272/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2273/// <2, 2, 3, 3> 2274bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2275 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2276 2277 unsigned NumElems = N->getNumOperands(); 2278 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2279 return false; 2280 2281 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2282 SDOperand BitI = N->getOperand(i); 2283 SDOperand BitI1 = N->getOperand(i + 1); 2284 2285 if (!isUndefOrEqual(BitI, j)) 2286 return false; 2287 if (!isUndefOrEqual(BitI1, j)) 2288 return false; 2289 } 2290 2291 return true; 2292} 2293 2294/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2295/// specifies a shuffle of elements that is suitable for input to MOVSS, 2296/// MOVSD, and MOVD, i.e. setting the lowest element. 2297static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2298 if (NumElts != 2 && NumElts != 4) 2299 return false; 2300 2301 if (!isUndefOrEqual(Elts[0], NumElts)) 2302 return false; 2303 2304 for (unsigned i = 1; i < NumElts; ++i) { 2305 if (!isUndefOrEqual(Elts[i], i)) 2306 return false; 2307 } 2308 2309 return true; 2310} 2311 2312bool X86::isMOVLMask(SDNode *N) { 2313 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2314 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2315} 2316 2317/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2318/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2319/// element of vector 2 and the other elements to come from vector 1 in order. 2320static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2321 bool V2IsSplat = false, 2322 bool V2IsUndef = false) { 2323 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2324 return false; 2325 2326 if (!isUndefOrEqual(Ops[0], 0)) 2327 return false; 2328 2329 for (unsigned i = 1; i < NumOps; ++i) { 2330 SDOperand Arg = Ops[i]; 2331 if (!(isUndefOrEqual(Arg, i+NumOps) || 2332 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2333 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2334 return false; 2335 } 2336 2337 return true; 2338} 2339 2340static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2341 bool V2IsUndef = false) { 2342 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2343 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2344 V2IsSplat, V2IsUndef); 2345} 2346 2347/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2348/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2349bool X86::isMOVSHDUPMask(SDNode *N) { 2350 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2351 2352 if (N->getNumOperands() != 4) 2353 return false; 2354 2355 // Expect 1, 1, 3, 3 2356 for (unsigned i = 0; i < 2; ++i) { 2357 SDOperand Arg = N->getOperand(i); 2358 if (Arg.getOpcode() == ISD::UNDEF) continue; 2359 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2360 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2361 if (Val != 1) return false; 2362 } 2363 2364 bool HasHi = false; 2365 for (unsigned i = 2; i < 4; ++i) { 2366 SDOperand Arg = N->getOperand(i); 2367 if (Arg.getOpcode() == ISD::UNDEF) continue; 2368 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2369 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2370 if (Val != 3) return false; 2371 HasHi = true; 2372 } 2373 2374 // Don't use movshdup if it can be done with a shufps. 2375 return HasHi; 2376} 2377 2378/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2379/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2380bool X86::isMOVSLDUPMask(SDNode *N) { 2381 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2382 2383 if (N->getNumOperands() != 4) 2384 return false; 2385 2386 // Expect 0, 0, 2, 2 2387 for (unsigned i = 0; i < 2; ++i) { 2388 SDOperand Arg = N->getOperand(i); 2389 if (Arg.getOpcode() == ISD::UNDEF) continue; 2390 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2391 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2392 if (Val != 0) return false; 2393 } 2394 2395 bool HasHi = false; 2396 for (unsigned i = 2; i < 4; ++i) { 2397 SDOperand Arg = N->getOperand(i); 2398 if (Arg.getOpcode() == ISD::UNDEF) continue; 2399 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2400 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2401 if (Val != 2) return false; 2402 HasHi = true; 2403 } 2404 2405 // Don't use movshdup if it can be done with a shufps. 2406 return HasHi; 2407} 2408 2409/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2410/// specifies a identity operation on the LHS or RHS. 2411static bool isIdentityMask(SDNode *N, bool RHS = false) { 2412 unsigned NumElems = N->getNumOperands(); 2413 for (unsigned i = 0; i < NumElems; ++i) 2414 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2415 return false; 2416 return true; 2417} 2418 2419/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2420/// a splat of a single element. 2421static bool isSplatMask(SDNode *N) { 2422 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2423 2424 // This is a splat operation if each element of the permute is the same, and 2425 // if the value doesn't reference the second vector. 2426 unsigned NumElems = N->getNumOperands(); 2427 SDOperand ElementBase; 2428 unsigned i = 0; 2429 for (; i != NumElems; ++i) { 2430 SDOperand Elt = N->getOperand(i); 2431 if (isa<ConstantSDNode>(Elt)) { 2432 ElementBase = Elt; 2433 break; 2434 } 2435 } 2436 2437 if (!ElementBase.Val) 2438 return false; 2439 2440 for (; i != NumElems; ++i) { 2441 SDOperand Arg = N->getOperand(i); 2442 if (Arg.getOpcode() == ISD::UNDEF) continue; 2443 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2444 if (Arg != ElementBase) return false; 2445 } 2446 2447 // Make sure it is a splat of the first vector operand. 2448 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2449} 2450 2451/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2452/// a splat of a single element and it's a 2 or 4 element mask. 2453bool X86::isSplatMask(SDNode *N) { 2454 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2455 2456 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2457 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2458 return false; 2459 return ::isSplatMask(N); 2460} 2461 2462/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2463/// specifies a splat of zero element. 2464bool X86::isSplatLoMask(SDNode *N) { 2465 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2466 2467 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2468 if (!isUndefOrEqual(N->getOperand(i), 0)) 2469 return false; 2470 return true; 2471} 2472 2473/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2474/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2475/// instructions. 2476unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2477 unsigned NumOperands = N->getNumOperands(); 2478 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2479 unsigned Mask = 0; 2480 for (unsigned i = 0; i < NumOperands; ++i) { 2481 unsigned Val = 0; 2482 SDOperand Arg = N->getOperand(NumOperands-i-1); 2483 if (Arg.getOpcode() != ISD::UNDEF) 2484 Val = cast<ConstantSDNode>(Arg)->getValue(); 2485 if (Val >= NumOperands) Val -= NumOperands; 2486 Mask |= Val; 2487 if (i != NumOperands - 1) 2488 Mask <<= Shift; 2489 } 2490 2491 return Mask; 2492} 2493 2494/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2495/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2496/// instructions. 2497unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2498 unsigned Mask = 0; 2499 // 8 nodes, but we only care about the last 4. 2500 for (unsigned i = 7; i >= 4; --i) { 2501 unsigned Val = 0; 2502 SDOperand Arg = N->getOperand(i); 2503 if (Arg.getOpcode() != ISD::UNDEF) 2504 Val = cast<ConstantSDNode>(Arg)->getValue(); 2505 Mask |= (Val - 4); 2506 if (i != 4) 2507 Mask <<= 2; 2508 } 2509 2510 return Mask; 2511} 2512 2513/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2514/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2515/// instructions. 2516unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2517 unsigned Mask = 0; 2518 // 8 nodes, but we only care about the first 4. 2519 for (int i = 3; i >= 0; --i) { 2520 unsigned Val = 0; 2521 SDOperand Arg = N->getOperand(i); 2522 if (Arg.getOpcode() != ISD::UNDEF) 2523 Val = cast<ConstantSDNode>(Arg)->getValue(); 2524 Mask |= Val; 2525 if (i != 0) 2526 Mask <<= 2; 2527 } 2528 2529 return Mask; 2530} 2531 2532/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2533/// specifies a 8 element shuffle that can be broken into a pair of 2534/// PSHUFHW and PSHUFLW. 2535static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2536 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2537 2538 if (N->getNumOperands() != 8) 2539 return false; 2540 2541 // Lower quadword shuffled. 2542 for (unsigned i = 0; i != 4; ++i) { 2543 SDOperand Arg = N->getOperand(i); 2544 if (Arg.getOpcode() == ISD::UNDEF) continue; 2545 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2546 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2547 if (Val >= 4) 2548 return false; 2549 } 2550 2551 // Upper quadword shuffled. 2552 for (unsigned i = 4; i != 8; ++i) { 2553 SDOperand Arg = N->getOperand(i); 2554 if (Arg.getOpcode() == ISD::UNDEF) continue; 2555 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2556 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2557 if (Val < 4 || Val > 7) 2558 return false; 2559 } 2560 2561 return true; 2562} 2563 2564/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2565/// values in ther permute mask. 2566static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1, 2567 SDOperand &V2, SDOperand &Mask, 2568 SelectionDAG &DAG) { 2569 MVT VT = Op.getValueType(); 2570 MVT MaskVT = Mask.getValueType(); 2571 MVT EltVT = MaskVT.getVectorElementType(); 2572 unsigned NumElems = Mask.getNumOperands(); 2573 SmallVector<SDOperand, 8> MaskVec; 2574 2575 for (unsigned i = 0; i != NumElems; ++i) { 2576 SDOperand Arg = Mask.getOperand(i); 2577 if (Arg.getOpcode() == ISD::UNDEF) { 2578 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2579 continue; 2580 } 2581 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2582 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2583 if (Val < NumElems) 2584 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2585 else 2586 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2587 } 2588 2589 std::swap(V1, V2); 2590 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2591 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2592} 2593 2594/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2595/// the two vector operands have swapped position. 2596static 2597SDOperand CommuteVectorShuffleMask(SDOperand Mask, SelectionDAG &DAG) { 2598 MVT MaskVT = Mask.getValueType(); 2599 MVT EltVT = MaskVT.getVectorElementType(); 2600 unsigned NumElems = Mask.getNumOperands(); 2601 SmallVector<SDOperand, 8> MaskVec; 2602 for (unsigned i = 0; i != NumElems; ++i) { 2603 SDOperand Arg = Mask.getOperand(i); 2604 if (Arg.getOpcode() == ISD::UNDEF) { 2605 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2606 continue; 2607 } 2608 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2609 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2610 if (Val < NumElems) 2611 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2612 else 2613 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2614 } 2615 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2616} 2617 2618 2619/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2620/// match movhlps. The lower half elements should come from upper half of 2621/// V1 (and in order), and the upper half elements should come from the upper 2622/// half of V2 (and in order). 2623static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2624 unsigned NumElems = Mask->getNumOperands(); 2625 if (NumElems != 4) 2626 return false; 2627 for (unsigned i = 0, e = 2; i != e; ++i) 2628 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2629 return false; 2630 for (unsigned i = 2; i != 4; ++i) 2631 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2632 return false; 2633 return true; 2634} 2635 2636/// isScalarLoadToVector - Returns true if the node is a scalar load that 2637/// is promoted to a vector. It also returns the LoadSDNode by reference if 2638/// required. 2639static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2640 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2641 N = N->getOperand(0).Val; 2642 if (ISD::isNON_EXTLoad(N)) { 2643 if (LD) 2644 *LD = cast<LoadSDNode>(N); 2645 return true; 2646 } 2647 } 2648 return false; 2649} 2650 2651/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2652/// match movlp{s|d}. The lower half elements should come from lower half of 2653/// V1 (and in order), and the upper half elements should come from the upper 2654/// half of V2 (and in order). And since V1 will become the source of the 2655/// MOVLP, it must be either a vector load or a scalar load to vector. 2656static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2657 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2658 return false; 2659 // Is V2 is a vector load, don't do this transformation. We will try to use 2660 // load folding shufps op. 2661 if (ISD::isNON_EXTLoad(V2)) 2662 return false; 2663 2664 unsigned NumElems = Mask->getNumOperands(); 2665 if (NumElems != 2 && NumElems != 4) 2666 return false; 2667 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2668 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2669 return false; 2670 for (unsigned i = NumElems/2; i != NumElems; ++i) 2671 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2672 return false; 2673 return true; 2674} 2675 2676/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2677/// all the same. 2678static bool isSplatVector(SDNode *N) { 2679 if (N->getOpcode() != ISD::BUILD_VECTOR) 2680 return false; 2681 2682 SDOperand SplatValue = N->getOperand(0); 2683 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2684 if (N->getOperand(i) != SplatValue) 2685 return false; 2686 return true; 2687} 2688 2689/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2690/// to an undef. 2691static bool isUndefShuffle(SDNode *N) { 2692 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2693 return false; 2694 2695 SDOperand V1 = N->getOperand(0); 2696 SDOperand V2 = N->getOperand(1); 2697 SDOperand Mask = N->getOperand(2); 2698 unsigned NumElems = Mask.getNumOperands(); 2699 for (unsigned i = 0; i != NumElems; ++i) { 2700 SDOperand Arg = Mask.getOperand(i); 2701 if (Arg.getOpcode() != ISD::UNDEF) { 2702 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2703 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2704 return false; 2705 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2706 return false; 2707 } 2708 } 2709 return true; 2710} 2711 2712/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2713/// constant +0.0. 2714static inline bool isZeroNode(SDOperand Elt) { 2715 return ((isa<ConstantSDNode>(Elt) && 2716 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2717 (isa<ConstantFPSDNode>(Elt) && 2718 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2719} 2720 2721/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2722/// to an zero vector. 2723static bool isZeroShuffle(SDNode *N) { 2724 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2725 return false; 2726 2727 SDOperand V1 = N->getOperand(0); 2728 SDOperand V2 = N->getOperand(1); 2729 SDOperand Mask = N->getOperand(2); 2730 unsigned NumElems = Mask.getNumOperands(); 2731 for (unsigned i = 0; i != NumElems; ++i) { 2732 SDOperand Arg = Mask.getOperand(i); 2733 if (Arg.getOpcode() == ISD::UNDEF) 2734 continue; 2735 2736 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2737 if (Idx < NumElems) { 2738 unsigned Opc = V1.Val->getOpcode(); 2739 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.Val)) 2740 continue; 2741 if (Opc != ISD::BUILD_VECTOR || 2742 !isZeroNode(V1.Val->getOperand(Idx))) 2743 return false; 2744 } else if (Idx >= NumElems) { 2745 unsigned Opc = V2.Val->getOpcode(); 2746 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.Val)) 2747 continue; 2748 if (Opc != ISD::BUILD_VECTOR || 2749 !isZeroNode(V2.Val->getOperand(Idx - NumElems))) 2750 return false; 2751 } 2752 } 2753 return true; 2754} 2755 2756/// getZeroVector - Returns a vector of specified type with all zero elements. 2757/// 2758static SDOperand getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2759 assert(VT.isVector() && "Expected a vector type"); 2760 2761 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2762 // type. This ensures they get CSE'd. 2763 SDOperand Vec; 2764 if (VT.getSizeInBits() == 64) { // MMX 2765 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2766 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2767 } else if (HasSSE2) { // SSE2 2768 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2769 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2770 } else { // SSE1 2771 SDOperand Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2772 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2773 } 2774 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2775} 2776 2777/// getOnesVector - Returns a vector of specified type with all bits set. 2778/// 2779static SDOperand getOnesVector(MVT VT, SelectionDAG &DAG) { 2780 assert(VT.isVector() && "Expected a vector type"); 2781 2782 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2783 // type. This ensures they get CSE'd. 2784 SDOperand Cst = DAG.getTargetConstant(~0U, MVT::i32); 2785 SDOperand Vec; 2786 if (VT.getSizeInBits() == 64) // MMX 2787 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2788 else // SSE 2789 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2790 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2791} 2792 2793 2794/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2795/// that point to V2 points to its first element. 2796static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) { 2797 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2798 2799 bool Changed = false; 2800 SmallVector<SDOperand, 8> MaskVec; 2801 unsigned NumElems = Mask.getNumOperands(); 2802 for (unsigned i = 0; i != NumElems; ++i) { 2803 SDOperand Arg = Mask.getOperand(i); 2804 if (Arg.getOpcode() != ISD::UNDEF) { 2805 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2806 if (Val > NumElems) { 2807 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2808 Changed = true; 2809 } 2810 } 2811 MaskVec.push_back(Arg); 2812 } 2813 2814 if (Changed) 2815 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2816 &MaskVec[0], MaskVec.size()); 2817 return Mask; 2818} 2819 2820/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2821/// operation of specified width. 2822static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2823 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2824 MVT BaseVT = MaskVT.getVectorElementType(); 2825 2826 SmallVector<SDOperand, 8> MaskVec; 2827 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2828 for (unsigned i = 1; i != NumElems; ++i) 2829 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2830 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2831} 2832 2833/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2834/// of specified width. 2835static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2836 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2837 MVT BaseVT = MaskVT.getVectorElementType(); 2838 SmallVector<SDOperand, 8> MaskVec; 2839 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2840 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2841 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2842 } 2843 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2844} 2845 2846/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2847/// of specified width. 2848static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2849 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2850 MVT BaseVT = MaskVT.getVectorElementType(); 2851 unsigned Half = NumElems/2; 2852 SmallVector<SDOperand, 8> MaskVec; 2853 for (unsigned i = 0; i != Half; ++i) { 2854 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2855 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2856 } 2857 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2858} 2859 2860/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2861/// element #0 of a vector with the specified index, leaving the rest of the 2862/// elements in place. 2863static SDOperand getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2864 SelectionDAG &DAG) { 2865 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2866 MVT BaseVT = MaskVT.getVectorElementType(); 2867 SmallVector<SDOperand, 8> MaskVec; 2868 // Element #0 of the result gets the elt we are replacing. 2869 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2870 for (unsigned i = 1; i != NumElems; ++i) 2871 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2872 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2873} 2874 2875/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2876static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) { 2877 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2878 MVT VT = Op.getValueType(); 2879 if (PVT == VT) 2880 return Op; 2881 SDOperand V1 = Op.getOperand(0); 2882 SDOperand Mask = Op.getOperand(2); 2883 unsigned NumElems = Mask.getNumOperands(); 2884 // Special handling of v4f32 -> v4i32. 2885 if (VT != MVT::v4f32) { 2886 Mask = getUnpacklMask(NumElems, DAG); 2887 while (NumElems > 4) { 2888 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2889 NumElems >>= 1; 2890 } 2891 Mask = getZeroVector(MVT::v4i32, true, DAG); 2892 } 2893 2894 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2895 SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2896 DAG.getNode(ISD::UNDEF, PVT), Mask); 2897 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2898} 2899 2900/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2901/// vector of zero or undef vector. This produces a shuffle where the low 2902/// element of V2 is swizzled into the zero/undef vector, landing at element 2903/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2904static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx, 2905 bool isZero, bool HasSSE2, 2906 SelectionDAG &DAG) { 2907 MVT VT = V2.getValueType(); 2908 SDOperand V1 = isZero 2909 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2910 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2911 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2912 MVT EVT = MaskVT.getVectorElementType(); 2913 SmallVector<SDOperand, 16> MaskVec; 2914 for (unsigned i = 0; i != NumElems; ++i) 2915 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2916 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2917 else 2918 MaskVec.push_back(DAG.getConstant(i, EVT)); 2919 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2920 &MaskVec[0], MaskVec.size()); 2921 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2922} 2923 2924/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2925/// a shuffle that is zero. 2926static 2927unsigned getNumOfConsecutiveZeros(SDOperand Op, SDOperand Mask, 2928 unsigned NumElems, bool Low, 2929 SelectionDAG &DAG) { 2930 unsigned NumZeros = 0; 2931 for (unsigned i = 0; i < NumElems; ++i) { 2932 unsigned Index = Low ? i : NumElems-i-1; 2933 SDOperand Idx = Mask.getOperand(Index); 2934 if (Idx.getOpcode() == ISD::UNDEF) { 2935 ++NumZeros; 2936 continue; 2937 } 2938 SDOperand Elt = DAG.getShuffleScalarElt(Op.Val, Index); 2939 if (Elt.Val && isZeroNode(Elt)) 2940 ++NumZeros; 2941 else 2942 break; 2943 } 2944 return NumZeros; 2945} 2946 2947/// isVectorShift - Returns true if the shuffle can be implemented as a 2948/// logical left or right shift of a vector. 2949static bool isVectorShift(SDOperand Op, SDOperand Mask, SelectionDAG &DAG, 2950 bool &isLeft, SDOperand &ShVal, unsigned &ShAmt) { 2951 unsigned NumElems = Mask.getNumOperands(); 2952 2953 isLeft = true; 2954 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 2955 if (!NumZeros) { 2956 isLeft = false; 2957 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 2958 if (!NumZeros) 2959 return false; 2960 } 2961 2962 bool SeenV1 = false; 2963 bool SeenV2 = false; 2964 for (unsigned i = NumZeros; i < NumElems; ++i) { 2965 unsigned Val = isLeft ? (i - NumZeros) : i; 2966 SDOperand Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 2967 if (Idx.getOpcode() == ISD::UNDEF) 2968 continue; 2969 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 2970 if (Index < NumElems) 2971 SeenV1 = true; 2972 else { 2973 Index -= NumElems; 2974 SeenV2 = true; 2975 } 2976 if (Index != Val) 2977 return false; 2978 } 2979 if (SeenV1 && SeenV2) 2980 return false; 2981 2982 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 2983 ShAmt = NumZeros; 2984 return true; 2985} 2986 2987 2988/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 2989/// 2990static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros, 2991 unsigned NumNonZero, unsigned NumZero, 2992 SelectionDAG &DAG, TargetLowering &TLI) { 2993 if (NumNonZero > 8) 2994 return SDOperand(); 2995 2996 SDOperand V(0, 0); 2997 bool First = true; 2998 for (unsigned i = 0; i < 16; ++i) { 2999 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3000 if (ThisIsNonZero && First) { 3001 if (NumZero) 3002 V = getZeroVector(MVT::v8i16, true, DAG); 3003 else 3004 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3005 First = false; 3006 } 3007 3008 if ((i & 1) != 0) { 3009 SDOperand ThisElt(0, 0), LastElt(0, 0); 3010 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3011 if (LastIsNonZero) { 3012 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3013 } 3014 if (ThisIsNonZero) { 3015 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3016 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3017 ThisElt, DAG.getConstant(8, MVT::i8)); 3018 if (LastIsNonZero) 3019 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3020 } else 3021 ThisElt = LastElt; 3022 3023 if (ThisElt.Val) 3024 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3025 DAG.getIntPtrConstant(i/2)); 3026 } 3027 } 3028 3029 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3030} 3031 3032/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3033/// 3034static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros, 3035 unsigned NumNonZero, unsigned NumZero, 3036 SelectionDAG &DAG, TargetLowering &TLI) { 3037 if (NumNonZero > 4) 3038 return SDOperand(); 3039 3040 SDOperand V(0, 0); 3041 bool First = true; 3042 for (unsigned i = 0; i < 8; ++i) { 3043 bool isNonZero = (NonZeros & (1 << i)) != 0; 3044 if (isNonZero) { 3045 if (First) { 3046 if (NumZero) 3047 V = getZeroVector(MVT::v8i16, true, DAG); 3048 else 3049 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3050 First = false; 3051 } 3052 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3053 DAG.getIntPtrConstant(i)); 3054 } 3055 } 3056 3057 return V; 3058} 3059 3060/// getVShift - Return a vector logical shift node. 3061/// 3062static SDOperand getVShift(bool isLeft, MVT VT, SDOperand SrcOp, 3063 unsigned NumBits, SelectionDAG &DAG, 3064 const TargetLowering &TLI) { 3065 bool isMMX = VT.getSizeInBits() == 64; 3066 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3067 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3068 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3069 return DAG.getNode(ISD::BIT_CONVERT, VT, 3070 DAG.getNode(Opc, ShVT, SrcOp, 3071 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3072} 3073 3074SDOperand 3075X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) { 3076 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3077 if (ISD::isBuildVectorAllZeros(Op.Val) || ISD::isBuildVectorAllOnes(Op.Val)) { 3078 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3079 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3080 // eliminated on x86-32 hosts. 3081 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3082 return Op; 3083 3084 if (ISD::isBuildVectorAllOnes(Op.Val)) 3085 return getOnesVector(Op.getValueType(), DAG); 3086 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3087 } 3088 3089 MVT VT = Op.getValueType(); 3090 MVT EVT = VT.getVectorElementType(); 3091 unsigned EVTBits = EVT.getSizeInBits(); 3092 3093 unsigned NumElems = Op.getNumOperands(); 3094 unsigned NumZero = 0; 3095 unsigned NumNonZero = 0; 3096 unsigned NonZeros = 0; 3097 bool IsAllConstants = true; 3098 SmallSet<SDOperand, 8> Values; 3099 for (unsigned i = 0; i < NumElems; ++i) { 3100 SDOperand Elt = Op.getOperand(i); 3101 if (Elt.getOpcode() == ISD::UNDEF) 3102 continue; 3103 Values.insert(Elt); 3104 if (Elt.getOpcode() != ISD::Constant && 3105 Elt.getOpcode() != ISD::ConstantFP) 3106 IsAllConstants = false; 3107 if (isZeroNode(Elt)) 3108 NumZero++; 3109 else { 3110 NonZeros |= (1 << i); 3111 NumNonZero++; 3112 } 3113 } 3114 3115 if (NumNonZero == 0) { 3116 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3117 return DAG.getNode(ISD::UNDEF, VT); 3118 } 3119 3120 // Special case for single non-zero, non-undef, element. 3121 if (NumNonZero == 1 && NumElems <= 4) { 3122 unsigned Idx = CountTrailingZeros_32(NonZeros); 3123 SDOperand Item = Op.getOperand(Idx); 3124 3125 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3126 // the value are obviously zero, truncate the value to i32 and do the 3127 // insertion that way. Only do this if the value is non-constant or if the 3128 // value is a constant being inserted into element 0. It is cheaper to do 3129 // a constant pool load than it is to do a movd + shuffle. 3130 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3131 (!IsAllConstants || Idx == 0)) { 3132 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3133 // Handle MMX and SSE both. 3134 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3135 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3136 3137 // Truncate the value (which may itself be a constant) to i32, and 3138 // convert it to a vector with movd (S2V+shuffle to zero extend). 3139 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3140 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3141 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3142 Subtarget->hasSSE2(), DAG); 3143 3144 // Now we have our 32-bit value zero extended in the low element of 3145 // a vector. If Idx != 0, swizzle it into place. 3146 if (Idx != 0) { 3147 SDOperand Ops[] = { 3148 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3149 getSwapEltZeroMask(VecElts, Idx, DAG) 3150 }; 3151 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3152 } 3153 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3154 } 3155 } 3156 3157 // If we have a constant or non-constant insertion into the low element of 3158 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3159 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3160 // depending on what the source datatype is. Because we can only get here 3161 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3162 if (Idx == 0 && 3163 // Don't do this for i64 values on x86-32. 3164 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3165 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3166 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3167 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3168 Subtarget->hasSSE2(), DAG); 3169 } 3170 3171 // Is it a vector logical left shift? 3172 if (NumElems == 2 && Idx == 1 && 3173 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3174 unsigned NumBits = VT.getSizeInBits(); 3175 return getVShift(true, VT, 3176 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3177 NumBits/2, DAG, *this); 3178 } 3179 3180 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3181 return SDOperand(); 3182 3183 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3184 // is a non-constant being inserted into an element other than the low one, 3185 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3186 // movd/movss) to move this into the low element, then shuffle it into 3187 // place. 3188 if (EVTBits == 32) { 3189 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3190 3191 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3192 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3193 Subtarget->hasSSE2(), DAG); 3194 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3195 MVT MaskEVT = MaskVT.getVectorElementType(); 3196 SmallVector<SDOperand, 8> MaskVec; 3197 for (unsigned i = 0; i < NumElems; i++) 3198 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3199 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3200 &MaskVec[0], MaskVec.size()); 3201 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3202 DAG.getNode(ISD::UNDEF, VT), Mask); 3203 } 3204 } 3205 3206 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3207 if (Values.size() == 1) 3208 return SDOperand(); 3209 3210 // A vector full of immediates; various special cases are already 3211 // handled, so this is best done with a single constant-pool load. 3212 if (IsAllConstants) 3213 return SDOperand(); 3214 3215 // Let legalizer expand 2-wide build_vectors. 3216 if (EVTBits == 64) { 3217 if (NumNonZero == 1) { 3218 // One half is zero or undef. 3219 unsigned Idx = CountTrailingZeros_32(NonZeros); 3220 SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3221 Op.getOperand(Idx)); 3222 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3223 Subtarget->hasSSE2(), DAG); 3224 } 3225 return SDOperand(); 3226 } 3227 3228 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3229 if (EVTBits == 8 && NumElems == 16) { 3230 SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3231 *this); 3232 if (V.Val) return V; 3233 } 3234 3235 if (EVTBits == 16 && NumElems == 8) { 3236 SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3237 *this); 3238 if (V.Val) return V; 3239 } 3240 3241 // If element VT is == 32 bits, turn it into a number of shuffles. 3242 SmallVector<SDOperand, 8> V; 3243 V.resize(NumElems); 3244 if (NumElems == 4 && NumZero > 0) { 3245 for (unsigned i = 0; i < 4; ++i) { 3246 bool isZero = !(NonZeros & (1 << i)); 3247 if (isZero) 3248 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3249 else 3250 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3251 } 3252 3253 for (unsigned i = 0; i < 2; ++i) { 3254 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3255 default: break; 3256 case 0: 3257 V[i] = V[i*2]; // Must be a zero vector. 3258 break; 3259 case 1: 3260 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3261 getMOVLMask(NumElems, DAG)); 3262 break; 3263 case 2: 3264 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3265 getMOVLMask(NumElems, DAG)); 3266 break; 3267 case 3: 3268 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3269 getUnpacklMask(NumElems, DAG)); 3270 break; 3271 } 3272 } 3273 3274 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3275 MVT EVT = MaskVT.getVectorElementType(); 3276 SmallVector<SDOperand, 8> MaskVec; 3277 bool Reverse = (NonZeros & 0x3) == 2; 3278 for (unsigned i = 0; i < 2; ++i) 3279 if (Reverse) 3280 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3281 else 3282 MaskVec.push_back(DAG.getConstant(i, EVT)); 3283 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3284 for (unsigned i = 0; i < 2; ++i) 3285 if (Reverse) 3286 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3287 else 3288 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3289 SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3290 &MaskVec[0], MaskVec.size()); 3291 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3292 } 3293 3294 if (Values.size() > 2) { 3295 // Expand into a number of unpckl*. 3296 // e.g. for v4f32 3297 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3298 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3299 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3300 SDOperand UnpckMask = getUnpacklMask(NumElems, DAG); 3301 for (unsigned i = 0; i < NumElems; ++i) 3302 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3303 NumElems >>= 1; 3304 while (NumElems != 0) { 3305 for (unsigned i = 0; i < NumElems; ++i) 3306 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3307 UnpckMask); 3308 NumElems >>= 1; 3309 } 3310 return V[0]; 3311 } 3312 3313 return SDOperand(); 3314} 3315 3316static 3317SDOperand LowerVECTOR_SHUFFLEv8i16(SDOperand V1, SDOperand V2, 3318 SDOperand PermMask, SelectionDAG &DAG, 3319 TargetLowering &TLI) { 3320 SDOperand NewV; 3321 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3322 MVT MaskEVT = MaskVT.getVectorElementType(); 3323 MVT PtrVT = TLI.getPointerTy(); 3324 SmallVector<SDOperand, 8> MaskElts(PermMask.Val->op_begin(), 3325 PermMask.Val->op_end()); 3326 3327 // First record which half of which vector the low elements come from. 3328 SmallVector<unsigned, 4> LowQuad(4); 3329 for (unsigned i = 0; i < 4; ++i) { 3330 SDOperand Elt = MaskElts[i]; 3331 if (Elt.getOpcode() == ISD::UNDEF) 3332 continue; 3333 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3334 int QuadIdx = EltIdx / 4; 3335 ++LowQuad[QuadIdx]; 3336 } 3337 int BestLowQuad = -1; 3338 unsigned MaxQuad = 1; 3339 for (unsigned i = 0; i < 4; ++i) { 3340 if (LowQuad[i] > MaxQuad) { 3341 BestLowQuad = i; 3342 MaxQuad = LowQuad[i]; 3343 } 3344 } 3345 3346 // Record which half of which vector the high elements come from. 3347 SmallVector<unsigned, 4> HighQuad(4); 3348 for (unsigned i = 4; i < 8; ++i) { 3349 SDOperand Elt = MaskElts[i]; 3350 if (Elt.getOpcode() == ISD::UNDEF) 3351 continue; 3352 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3353 int QuadIdx = EltIdx / 4; 3354 ++HighQuad[QuadIdx]; 3355 } 3356 int BestHighQuad = -1; 3357 MaxQuad = 1; 3358 for (unsigned i = 0; i < 4; ++i) { 3359 if (HighQuad[i] > MaxQuad) { 3360 BestHighQuad = i; 3361 MaxQuad = HighQuad[i]; 3362 } 3363 } 3364 3365 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3366 if (BestLowQuad != -1 || BestHighQuad != -1) { 3367 // First sort the 4 chunks in order using shufpd. 3368 SmallVector<SDOperand, 8> MaskVec; 3369 if (BestLowQuad != -1) 3370 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3371 else 3372 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3373 if (BestHighQuad != -1) 3374 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3375 else 3376 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3377 SDOperand Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3378 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3379 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3380 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3381 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3382 3383 // Now sort high and low parts separately. 3384 BitVector InOrder(8); 3385 if (BestLowQuad != -1) { 3386 // Sort lower half in order using PSHUFLW. 3387 MaskVec.clear(); 3388 bool AnyOutOrder = false; 3389 for (unsigned i = 0; i != 4; ++i) { 3390 SDOperand Elt = MaskElts[i]; 3391 if (Elt.getOpcode() == ISD::UNDEF) { 3392 MaskVec.push_back(Elt); 3393 InOrder.set(i); 3394 } else { 3395 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3396 if (EltIdx != i) 3397 AnyOutOrder = true; 3398 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3399 // If this element is in the right place after this shuffle, then 3400 // remember it. 3401 if ((int)(EltIdx / 4) == BestLowQuad) 3402 InOrder.set(i); 3403 } 3404 } 3405 if (AnyOutOrder) { 3406 for (unsigned i = 4; i != 8; ++i) 3407 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3408 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3409 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3410 } 3411 } 3412 3413 if (BestHighQuad != -1) { 3414 // Sort high half in order using PSHUFHW if possible. 3415 MaskVec.clear(); 3416 for (unsigned i = 0; i != 4; ++i) 3417 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3418 bool AnyOutOrder = false; 3419 for (unsigned i = 4; i != 8; ++i) { 3420 SDOperand Elt = MaskElts[i]; 3421 if (Elt.getOpcode() == ISD::UNDEF) { 3422 MaskVec.push_back(Elt); 3423 InOrder.set(i); 3424 } else { 3425 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3426 if (EltIdx != i) 3427 AnyOutOrder = true; 3428 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3429 // If this element is in the right place after this shuffle, then 3430 // remember it. 3431 if ((int)(EltIdx / 4) == BestHighQuad) 3432 InOrder.set(i); 3433 } 3434 } 3435 if (AnyOutOrder) { 3436 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3437 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3438 } 3439 } 3440 3441 // The other elements are put in the right place using pextrw and pinsrw. 3442 for (unsigned i = 0; i != 8; ++i) { 3443 if (InOrder[i]) 3444 continue; 3445 SDOperand Elt = MaskElts[i]; 3446 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3447 SDOperand ExtOp = (EltIdx < 8) 3448 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3449 DAG.getConstant(EltIdx, PtrVT)) 3450 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3451 DAG.getConstant(EltIdx - 8, PtrVT)); 3452 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3453 DAG.getConstant(i, PtrVT)); 3454 } 3455 return NewV; 3456 } 3457 3458 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use 3459 ///as few as possible. 3460 // First, let's find out how many elements are already in the right order. 3461 unsigned V1InOrder = 0; 3462 unsigned V1FromV1 = 0; 3463 unsigned V2InOrder = 0; 3464 unsigned V2FromV2 = 0; 3465 SmallVector<SDOperand, 8> V1Elts; 3466 SmallVector<SDOperand, 8> V2Elts; 3467 for (unsigned i = 0; i < 8; ++i) { 3468 SDOperand Elt = MaskElts[i]; 3469 if (Elt.getOpcode() == ISD::UNDEF) { 3470 V1Elts.push_back(Elt); 3471 V2Elts.push_back(Elt); 3472 ++V1InOrder; 3473 ++V2InOrder; 3474 continue; 3475 } 3476 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3477 if (EltIdx == i) { 3478 V1Elts.push_back(Elt); 3479 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3480 ++V1InOrder; 3481 } else if (EltIdx == i+8) { 3482 V1Elts.push_back(Elt); 3483 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3484 ++V2InOrder; 3485 } else if (EltIdx < 8) { 3486 V1Elts.push_back(Elt); 3487 ++V1FromV1; 3488 } else { 3489 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3490 ++V2FromV2; 3491 } 3492 } 3493 3494 if (V2InOrder > V1InOrder) { 3495 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3496 std::swap(V1, V2); 3497 std::swap(V1Elts, V2Elts); 3498 std::swap(V1FromV1, V2FromV2); 3499 } 3500 3501 if ((V1FromV1 + V1InOrder) != 8) { 3502 // Some elements are from V2. 3503 if (V1FromV1) { 3504 // If there are elements that are from V1 but out of place, 3505 // then first sort them in place 3506 SmallVector<SDOperand, 8> MaskVec; 3507 for (unsigned i = 0; i < 8; ++i) { 3508 SDOperand Elt = V1Elts[i]; 3509 if (Elt.getOpcode() == ISD::UNDEF) { 3510 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3511 continue; 3512 } 3513 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3514 if (EltIdx >= 8) 3515 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3516 else 3517 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3518 } 3519 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3520 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3521 } 3522 3523 NewV = V1; 3524 for (unsigned i = 0; i < 8; ++i) { 3525 SDOperand Elt = V1Elts[i]; 3526 if (Elt.getOpcode() == ISD::UNDEF) 3527 continue; 3528 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3529 if (EltIdx < 8) 3530 continue; 3531 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3532 DAG.getConstant(EltIdx - 8, PtrVT)); 3533 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3534 DAG.getConstant(i, PtrVT)); 3535 } 3536 return NewV; 3537 } else { 3538 // All elements are from V1. 3539 NewV = V1; 3540 for (unsigned i = 0; i < 8; ++i) { 3541 SDOperand Elt = V1Elts[i]; 3542 if (Elt.getOpcode() == ISD::UNDEF) 3543 continue; 3544 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3545 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3546 DAG.getConstant(EltIdx, PtrVT)); 3547 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3548 DAG.getConstant(i, PtrVT)); 3549 } 3550 return NewV; 3551 } 3552} 3553 3554/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3555/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3556/// done when every pair / quad of shuffle mask elements point to elements in 3557/// the right sequence. e.g. 3558/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3559static 3560SDOperand RewriteAsNarrowerShuffle(SDOperand V1, SDOperand V2, 3561 MVT VT, 3562 SDOperand PermMask, SelectionDAG &DAG, 3563 TargetLowering &TLI) { 3564 unsigned NumElems = PermMask.getNumOperands(); 3565 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3566 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3567 MVT NewVT = MaskVT; 3568 switch (VT.getSimpleVT()) { 3569 default: assert(false && "Unexpected!"); 3570 case MVT::v4f32: NewVT = MVT::v2f64; break; 3571 case MVT::v4i32: NewVT = MVT::v2i64; break; 3572 case MVT::v8i16: NewVT = MVT::v4i32; break; 3573 case MVT::v16i8: NewVT = MVT::v4i32; break; 3574 } 3575 3576 if (NewWidth == 2) { 3577 if (VT.isInteger()) 3578 NewVT = MVT::v2i64; 3579 else 3580 NewVT = MVT::v2f64; 3581 } 3582 unsigned Scale = NumElems / NewWidth; 3583 SmallVector<SDOperand, 8> MaskVec; 3584 for (unsigned i = 0; i < NumElems; i += Scale) { 3585 unsigned StartIdx = ~0U; 3586 for (unsigned j = 0; j < Scale; ++j) { 3587 SDOperand Elt = PermMask.getOperand(i+j); 3588 if (Elt.getOpcode() == ISD::UNDEF) 3589 continue; 3590 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3591 if (StartIdx == ~0U) 3592 StartIdx = EltIdx - (EltIdx % Scale); 3593 if (EltIdx != StartIdx + j) 3594 return SDOperand(); 3595 } 3596 if (StartIdx == ~0U) 3597 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MVT::i32)); 3598 else 3599 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MVT::i32)); 3600 } 3601 3602 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3603 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3604 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3605 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3606 &MaskVec[0], MaskVec.size())); 3607} 3608 3609/// getVZextMovL - Return a zero-extending vector move low node. 3610/// 3611static SDOperand getVZextMovL(MVT VT, MVT OpVT, 3612 SDOperand SrcOp, SelectionDAG &DAG, 3613 const X86Subtarget *Subtarget) { 3614 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3615 LoadSDNode *LD = NULL; 3616 if (!isScalarLoadToVector(SrcOp.Val, &LD)) 3617 LD = dyn_cast<LoadSDNode>(SrcOp); 3618 if (!LD) { 3619 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3620 // instead. 3621 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3622 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3623 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3624 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3625 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3626 // PR2108 3627 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3628 return DAG.getNode(ISD::BIT_CONVERT, VT, 3629 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3630 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3631 SrcOp.getOperand(0).getOperand(0)))); 3632 } 3633 } 3634 } 3635 3636 return DAG.getNode(ISD::BIT_CONVERT, VT, 3637 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3638 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3639} 3640 3641SDOperand 3642X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) { 3643 SDOperand V1 = Op.getOperand(0); 3644 SDOperand V2 = Op.getOperand(1); 3645 SDOperand PermMask = Op.getOperand(2); 3646 MVT VT = Op.getValueType(); 3647 unsigned NumElems = PermMask.getNumOperands(); 3648 bool isMMX = VT.getSizeInBits() == 64; 3649 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3650 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3651 bool V1IsSplat = false; 3652 bool V2IsSplat = false; 3653 3654 if (isUndefShuffle(Op.Val)) 3655 return DAG.getNode(ISD::UNDEF, VT); 3656 3657 if (isZeroShuffle(Op.Val)) 3658 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3659 3660 if (isIdentityMask(PermMask.Val)) 3661 return V1; 3662 else if (isIdentityMask(PermMask.Val, true)) 3663 return V2; 3664 3665 if (isSplatMask(PermMask.Val)) { 3666 if (isMMX || NumElems < 4) return Op; 3667 // Promote it to a v4{if}32 splat. 3668 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3669 } 3670 3671 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3672 // do it! 3673 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3674 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3675 if (NewOp.Val) 3676 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3677 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3678 // FIXME: Figure out a cleaner way to do this. 3679 // Try to make use of movq to zero out the top part. 3680 if (ISD::isBuildVectorAllZeros(V2.Val)) { 3681 SDOperand NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3682 DAG, *this); 3683 if (NewOp.Val) { 3684 SDOperand NewV1 = NewOp.getOperand(0); 3685 SDOperand NewV2 = NewOp.getOperand(1); 3686 SDOperand NewMask = NewOp.getOperand(2); 3687 if (isCommutedMOVL(NewMask.Val, true, false)) { 3688 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3689 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3690 } 3691 } 3692 } else if (ISD::isBuildVectorAllZeros(V1.Val)) { 3693 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3694 DAG, *this); 3695 if (NewOp.Val && X86::isMOVLMask(NewOp.getOperand(2).Val)) 3696 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3697 DAG, Subtarget); 3698 } 3699 } 3700 3701 // Check if this can be converted into a logical shift. 3702 bool isLeft = false; 3703 unsigned ShAmt = 0; 3704 SDOperand ShVal; 3705 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3706 if (isShift && ShVal.hasOneUse()) { 3707 // If the shifted value has multiple uses, it may be cheaper to use 3708 // v_set0 + movlhps or movhlps, etc. 3709 MVT EVT = VT.getVectorElementType(); 3710 ShAmt *= EVT.getSizeInBits(); 3711 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3712 } 3713 3714 if (X86::isMOVLMask(PermMask.Val)) { 3715 if (V1IsUndef) 3716 return V2; 3717 if (ISD::isBuildVectorAllZeros(V1.Val)) 3718 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3719 return Op; 3720 } 3721 3722 if (X86::isMOVSHDUPMask(PermMask.Val) || 3723 X86::isMOVSLDUPMask(PermMask.Val) || 3724 X86::isMOVHLPSMask(PermMask.Val) || 3725 X86::isMOVHPMask(PermMask.Val) || 3726 X86::isMOVLPMask(PermMask.Val)) 3727 return Op; 3728 3729 if (ShouldXformToMOVHLPS(PermMask.Val) || 3730 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val)) 3731 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3732 3733 if (isShift) { 3734 // No better options. Use a vshl / vsrl. 3735 MVT EVT = VT.getVectorElementType(); 3736 ShAmt *= EVT.getSizeInBits(); 3737 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3738 } 3739 3740 bool Commuted = false; 3741 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3742 // 1,1,1,1 -> v8i16 though. 3743 V1IsSplat = isSplatVector(V1.Val); 3744 V2IsSplat = isSplatVector(V2.Val); 3745 3746 // Canonicalize the splat or undef, if present, to be on the RHS. 3747 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3748 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3749 std::swap(V1IsSplat, V2IsSplat); 3750 std::swap(V1IsUndef, V2IsUndef); 3751 Commuted = true; 3752 } 3753 3754 // FIXME: Figure out a cleaner way to do this. 3755 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) { 3756 if (V2IsUndef) return V1; 3757 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3758 if (V2IsSplat) { 3759 // V2 is a splat, so the mask may be malformed. That is, it may point 3760 // to any V2 element. The instruction selectior won't like this. Get 3761 // a corrected mask and commute to form a proper MOVS{S|D}. 3762 SDOperand NewMask = getMOVLMask(NumElems, DAG); 3763 if (NewMask.Val != PermMask.Val) 3764 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3765 } 3766 return Op; 3767 } 3768 3769 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3770 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3771 X86::isUNPCKLMask(PermMask.Val) || 3772 X86::isUNPCKHMask(PermMask.Val)) 3773 return Op; 3774 3775 if (V2IsSplat) { 3776 // Normalize mask so all entries that point to V2 points to its first 3777 // element then try to match unpck{h|l} again. If match, return a 3778 // new vector_shuffle with the corrected mask. 3779 SDOperand NewMask = NormalizeMask(PermMask, DAG); 3780 if (NewMask.Val != PermMask.Val) { 3781 if (X86::isUNPCKLMask(PermMask.Val, true)) { 3782 SDOperand NewMask = getUnpacklMask(NumElems, DAG); 3783 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3784 } else if (X86::isUNPCKHMask(PermMask.Val, true)) { 3785 SDOperand NewMask = getUnpackhMask(NumElems, DAG); 3786 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3787 } 3788 } 3789 } 3790 3791 // Normalize the node to match x86 shuffle ops if needed 3792 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val)) 3793 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3794 3795 if (Commuted) { 3796 // Commute is back and try unpck* again. 3797 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3798 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3799 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3800 X86::isUNPCKLMask(PermMask.Val) || 3801 X86::isUNPCKHMask(PermMask.Val)) 3802 return Op; 3803 } 3804 3805 // Try PSHUF* first, then SHUFP*. 3806 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 3807 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 3808 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.Val)) { 3809 if (V2.getOpcode() != ISD::UNDEF) 3810 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 3811 DAG.getNode(ISD::UNDEF, VT), PermMask); 3812 return Op; 3813 } 3814 3815 if (!isMMX) { 3816 if (Subtarget->hasSSE2() && 3817 (X86::isPSHUFDMask(PermMask.Val) || 3818 X86::isPSHUFHWMask(PermMask.Val) || 3819 X86::isPSHUFLWMask(PermMask.Val))) { 3820 MVT RVT = VT; 3821 if (VT == MVT::v4f32) { 3822 RVT = MVT::v4i32; 3823 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 3824 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 3825 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3826 } else if (V2.getOpcode() != ISD::UNDEF) 3827 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 3828 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3829 if (RVT != VT) 3830 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 3831 return Op; 3832 } 3833 3834 // Binary or unary shufps. 3835 if (X86::isSHUFPMask(PermMask.Val) || 3836 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.Val))) 3837 return Op; 3838 } 3839 3840 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 3841 if (VT == MVT::v8i16) { 3842 SDOperand NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 3843 if (NewOp.Val) 3844 return NewOp; 3845 } 3846 3847 // Handle all 4 wide cases with a number of shuffles. 3848 if (NumElems == 4 && !isMMX) { 3849 // Don't do this for MMX. 3850 MVT MaskVT = PermMask.getValueType(); 3851 MVT MaskEVT = MaskVT.getVectorElementType(); 3852 SmallVector<std::pair<int, int>, 8> Locs; 3853 Locs.reserve(NumElems); 3854 SmallVector<SDOperand, 8> Mask1(NumElems, 3855 DAG.getNode(ISD::UNDEF, MaskEVT)); 3856 SmallVector<SDOperand, 8> Mask2(NumElems, 3857 DAG.getNode(ISD::UNDEF, MaskEVT)); 3858 unsigned NumHi = 0; 3859 unsigned NumLo = 0; 3860 // If no more than two elements come from either vector. This can be 3861 // implemented with two shuffles. First shuffle gather the elements. 3862 // The second shuffle, which takes the first shuffle as both of its 3863 // vector operands, put the elements into the right order. 3864 for (unsigned i = 0; i != NumElems; ++i) { 3865 SDOperand Elt = PermMask.getOperand(i); 3866 if (Elt.getOpcode() == ISD::UNDEF) { 3867 Locs[i] = std::make_pair(-1, -1); 3868 } else { 3869 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3870 if (Val < NumElems) { 3871 Locs[i] = std::make_pair(0, NumLo); 3872 Mask1[NumLo] = Elt; 3873 NumLo++; 3874 } else { 3875 Locs[i] = std::make_pair(1, NumHi); 3876 if (2+NumHi < NumElems) 3877 Mask1[2+NumHi] = Elt; 3878 NumHi++; 3879 } 3880 } 3881 } 3882 if (NumLo <= 2 && NumHi <= 2) { 3883 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3884 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3885 &Mask1[0], Mask1.size())); 3886 for (unsigned i = 0; i != NumElems; ++i) { 3887 if (Locs[i].first == -1) 3888 continue; 3889 else { 3890 unsigned Idx = (i < NumElems/2) ? 0 : NumElems; 3891 Idx += Locs[i].first * (NumElems/2) + Locs[i].second; 3892 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3893 } 3894 } 3895 3896 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3897 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3898 &Mask2[0], Mask2.size())); 3899 } 3900 3901 // Break it into (shuffle shuffle_hi, shuffle_lo). 3902 Locs.clear(); 3903 SmallVector<SDOperand,8> LoMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); 3904 SmallVector<SDOperand,8> HiMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); 3905 SmallVector<SDOperand,8> *MaskPtr = &LoMask; 3906 unsigned MaskIdx = 0; 3907 unsigned LoIdx = 0; 3908 unsigned HiIdx = NumElems/2; 3909 for (unsigned i = 0; i != NumElems; ++i) { 3910 if (i == NumElems/2) { 3911 MaskPtr = &HiMask; 3912 MaskIdx = 1; 3913 LoIdx = 0; 3914 HiIdx = NumElems/2; 3915 } 3916 SDOperand Elt = PermMask.getOperand(i); 3917 if (Elt.getOpcode() == ISD::UNDEF) { 3918 Locs[i] = std::make_pair(-1, -1); 3919 } else if (cast<ConstantSDNode>(Elt)->getValue() < NumElems) { 3920 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3921 (*MaskPtr)[LoIdx] = Elt; 3922 LoIdx++; 3923 } else { 3924 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3925 (*MaskPtr)[HiIdx] = Elt; 3926 HiIdx++; 3927 } 3928 } 3929 3930 SDOperand LoShuffle = 3931 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3932 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3933 &LoMask[0], LoMask.size())); 3934 SDOperand HiShuffle = 3935 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3936 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3937 &HiMask[0], HiMask.size())); 3938 SmallVector<SDOperand, 8> MaskOps; 3939 for (unsigned i = 0; i != NumElems; ++i) { 3940 if (Locs[i].first == -1) { 3941 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3942 } else { 3943 unsigned Idx = Locs[i].first * NumElems + Locs[i].second; 3944 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3945 } 3946 } 3947 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3948 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3949 &MaskOps[0], MaskOps.size())); 3950 } 3951 3952 return SDOperand(); 3953} 3954 3955SDOperand 3956X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDOperand Op, 3957 SelectionDAG &DAG) { 3958 MVT VT = Op.getValueType(); 3959 if (VT.getSizeInBits() == 8) { 3960 SDOperand Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 3961 Op.getOperand(0), Op.getOperand(1)); 3962 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 3963 DAG.getValueType(VT)); 3964 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 3965 } else if (VT.getSizeInBits() == 16) { 3966 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 3967 Op.getOperand(0), Op.getOperand(1)); 3968 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 3969 DAG.getValueType(VT)); 3970 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 3971 } else if (VT == MVT::f32) { 3972 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 3973 // the result back to FR32 register. It's only worth matching if the 3974 // result has a single use which is a store or a bitcast to i32. 3975 if (!Op.hasOneUse()) 3976 return SDOperand(); 3977 SDNode *User = Op.Val->use_begin()->getUser(); 3978 if (User->getOpcode() != ISD::STORE && 3979 (User->getOpcode() != ISD::BIT_CONVERT || 3980 User->getValueType(0) != MVT::i32)) 3981 return SDOperand(); 3982 SDOperand Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 3983 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 3984 Op.getOperand(1)); 3985 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 3986 } 3987 return SDOperand(); 3988} 3989 3990 3991SDOperand 3992X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 3993 if (!isa<ConstantSDNode>(Op.getOperand(1))) 3994 return SDOperand(); 3995 3996 if (Subtarget->hasSSE41()) { 3997 SDOperand Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 3998 if (Res.Val) 3999 return Res; 4000 } 4001 4002 MVT VT = Op.getValueType(); 4003 // TODO: handle v16i8. 4004 if (VT.getSizeInBits() == 16) { 4005 SDOperand Vec = Op.getOperand(0); 4006 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4007 if (Idx == 0) 4008 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4009 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4010 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4011 Op.getOperand(1))); 4012 // Transform it so it match pextrw which produces a 32-bit result. 4013 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4014 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4015 Op.getOperand(0), Op.getOperand(1)); 4016 SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4017 DAG.getValueType(VT)); 4018 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4019 } else if (VT.getSizeInBits() == 32) { 4020 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4021 if (Idx == 0) 4022 return Op; 4023 // SHUFPS the element to the lowest double word, then movss. 4024 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4025 SmallVector<SDOperand, 8> IdxVec; 4026 IdxVec. 4027 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4028 IdxVec. 4029 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4030 IdxVec. 4031 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4032 IdxVec. 4033 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4034 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4035 &IdxVec[0], IdxVec.size()); 4036 SDOperand Vec = Op.getOperand(0); 4037 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4038 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4039 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4040 DAG.getIntPtrConstant(0)); 4041 } else if (VT.getSizeInBits() == 64) { 4042 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4043 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4044 // to match extract_elt for f64. 4045 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4046 if (Idx == 0) 4047 return Op; 4048 4049 // UNPCKHPD the element to the lowest double word, then movsd. 4050 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4051 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4052 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4053 SmallVector<SDOperand, 8> IdxVec; 4054 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4055 IdxVec. 4056 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4057 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4058 &IdxVec[0], IdxVec.size()); 4059 SDOperand Vec = Op.getOperand(0); 4060 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4061 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4062 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4063 DAG.getIntPtrConstant(0)); 4064 } 4065 4066 return SDOperand(); 4067} 4068 4069SDOperand 4070X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDOperand Op, SelectionDAG &DAG){ 4071 MVT VT = Op.getValueType(); 4072 MVT EVT = VT.getVectorElementType(); 4073 4074 SDOperand N0 = Op.getOperand(0); 4075 SDOperand N1 = Op.getOperand(1); 4076 SDOperand N2 = Op.getOperand(2); 4077 4078 if ((EVT.getSizeInBits() == 8) || (EVT.getSizeInBits() == 16)) { 4079 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4080 : X86ISD::PINSRW; 4081 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4082 // argument. 4083 if (N1.getValueType() != MVT::i32) 4084 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4085 if (N2.getValueType() != MVT::i32) 4086 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4087 return DAG.getNode(Opc, VT, N0, N1, N2); 4088 } else if (EVT == MVT::f32) { 4089 // Bits [7:6] of the constant are the source select. This will always be 4090 // zero here. The DAG Combiner may combine an extract_elt index into these 4091 // bits. For example (insert (extract, 3), 2) could be matched by putting 4092 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4093 // Bits [5:4] of the constant are the destination select. This is the 4094 // value of the incoming immediate. 4095 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4096 // combine either bitwise AND or insert of float 0.0 to set these bits. 4097 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4098 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4099 } 4100 return SDOperand(); 4101} 4102 4103SDOperand 4104X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 4105 MVT VT = Op.getValueType(); 4106 MVT EVT = VT.getVectorElementType(); 4107 4108 if (Subtarget->hasSSE41()) 4109 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4110 4111 if (EVT == MVT::i8) 4112 return SDOperand(); 4113 4114 SDOperand N0 = Op.getOperand(0); 4115 SDOperand N1 = Op.getOperand(1); 4116 SDOperand N2 = Op.getOperand(2); 4117 4118 if (EVT.getSizeInBits() == 16) { 4119 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4120 // as its second argument. 4121 if (N1.getValueType() != MVT::i32) 4122 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4123 if (N2.getValueType() != MVT::i32) 4124 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4125 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4126 } 4127 return SDOperand(); 4128} 4129 4130SDOperand 4131X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) { 4132 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4133 MVT VT = MVT::v2i32; 4134 switch (Op.getValueType().getSimpleVT()) { 4135 default: break; 4136 case MVT::v16i8: 4137 case MVT::v8i16: 4138 VT = MVT::v4i32; 4139 break; 4140 } 4141 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4142 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4143} 4144 4145// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4146// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4147// one of the above mentioned nodes. It has to be wrapped because otherwise 4148// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4149// be used to form addressing mode. These wrapped nodes will be selected 4150// into MOV32ri. 4151SDOperand 4152X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) { 4153 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4154 SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(), 4155 getPointerTy(), 4156 CP->getAlignment()); 4157 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4158 // With PIC, the address is actually $g + Offset. 4159 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4160 !Subtarget->isPICStyleRIPRel()) { 4161 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4162 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4163 Result); 4164 } 4165 4166 return Result; 4167} 4168 4169SDOperand 4170X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) { 4171 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4172 SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4173 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4174 // With PIC, the address is actually $g + Offset. 4175 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4176 !Subtarget->isPICStyleRIPRel()) { 4177 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4178 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4179 Result); 4180 } 4181 4182 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4183 // load the value at address GV, not the value of GV itself. This means that 4184 // the GlobalAddress must be in the base or index register of the address, not 4185 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4186 // The same applies for external symbols during PIC codegen 4187 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4188 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4189 PseudoSourceValue::getGOT(), 0); 4190 4191 return Result; 4192} 4193 4194// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4195static SDOperand 4196LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4197 const MVT PtrVT) { 4198 SDOperand InFlag; 4199 SDOperand Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4200 DAG.getNode(X86ISD::GlobalBaseReg, 4201 PtrVT), InFlag); 4202 InFlag = Chain.getValue(1); 4203 4204 // emit leal symbol@TLSGD(,%ebx,1), %eax 4205 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4206 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4207 GA->getValueType(0), 4208 GA->getOffset()); 4209 SDOperand Ops[] = { Chain, TGA, InFlag }; 4210 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4211 InFlag = Result.getValue(2); 4212 Chain = Result.getValue(1); 4213 4214 // call ___tls_get_addr. This function receives its argument in 4215 // the register EAX. 4216 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4217 InFlag = Chain.getValue(1); 4218 4219 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4220 SDOperand Ops1[] = { Chain, 4221 DAG.getTargetExternalSymbol("___tls_get_addr", 4222 PtrVT), 4223 DAG.getRegister(X86::EAX, PtrVT), 4224 DAG.getRegister(X86::EBX, PtrVT), 4225 InFlag }; 4226 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4227 InFlag = Chain.getValue(1); 4228 4229 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4230} 4231 4232// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4233static SDOperand 4234LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4235 const MVT PtrVT) { 4236 SDOperand InFlag, Chain; 4237 4238 // emit leaq symbol@TLSGD(%rip), %rdi 4239 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4240 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4241 GA->getValueType(0), 4242 GA->getOffset()); 4243 SDOperand Ops[] = { DAG.getEntryNode(), TGA}; 4244 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4245 Chain = Result.getValue(1); 4246 InFlag = Result.getValue(2); 4247 4248 // call ___tls_get_addr. This function receives its argument in 4249 // the register RDI. 4250 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4251 InFlag = Chain.getValue(1); 4252 4253 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4254 SDOperand Ops1[] = { Chain, 4255 DAG.getTargetExternalSymbol("___tls_get_addr", 4256 PtrVT), 4257 DAG.getRegister(X86::RDI, PtrVT), 4258 InFlag }; 4259 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4260 InFlag = Chain.getValue(1); 4261 4262 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4263} 4264 4265// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4266// "local exec" model. 4267static SDOperand LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4268 const MVT PtrVT) { 4269 // Get the Thread Pointer 4270 SDOperand ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4271 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4272 // exec) 4273 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4274 GA->getValueType(0), 4275 GA->getOffset()); 4276 SDOperand Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4277 4278 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4279 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4280 PseudoSourceValue::getGOT(), 0); 4281 4282 // The address of the thread local variable is the add of the thread 4283 // pointer with the offset of the variable. 4284 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4285} 4286 4287SDOperand 4288X86TargetLowering::LowerGlobalTLSAddress(SDOperand Op, SelectionDAG &DAG) { 4289 // TODO: implement the "local dynamic" model 4290 // TODO: implement the "initial exec"model for pic executables 4291 assert(Subtarget->isTargetELF() && 4292 "TLS not implemented for non-ELF targets"); 4293 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4294 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4295 // otherwise use the "Local Exec"TLS Model 4296 if (Subtarget->is64Bit()) { 4297 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4298 } else { 4299 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4300 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4301 else 4302 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4303 } 4304} 4305 4306SDOperand 4307X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) { 4308 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4309 SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4310 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4311 // With PIC, the address is actually $g + Offset. 4312 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4313 !Subtarget->isPICStyleRIPRel()) { 4314 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4315 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4316 Result); 4317 } 4318 4319 return Result; 4320} 4321 4322SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) { 4323 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4324 SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4325 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4326 // With PIC, the address is actually $g + Offset. 4327 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4328 !Subtarget->isPICStyleRIPRel()) { 4329 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4330 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4331 Result); 4332 } 4333 4334 return Result; 4335} 4336 4337/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4338/// take a 2 x i32 value to shift plus a shift amount. 4339SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) { 4340 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4341 MVT VT = Op.getValueType(); 4342 unsigned VTBits = VT.getSizeInBits(); 4343 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4344 SDOperand ShOpLo = Op.getOperand(0); 4345 SDOperand ShOpHi = Op.getOperand(1); 4346 SDOperand ShAmt = Op.getOperand(2); 4347 SDOperand Tmp1 = isSRA ? 4348 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4349 DAG.getConstant(0, VT); 4350 4351 SDOperand Tmp2, Tmp3; 4352 if (Op.getOpcode() == ISD::SHL_PARTS) { 4353 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4354 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4355 } else { 4356 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4357 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4358 } 4359 4360 SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4361 DAG.getConstant(VTBits, MVT::i8)); 4362 SDOperand Cond = DAG.getNode(X86ISD::CMP, VT, 4363 AndNode, DAG.getConstant(0, MVT::i8)); 4364 4365 SDOperand Hi, Lo; 4366 SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4367 SDOperand Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 4368 SDOperand Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 4369 4370 if (Op.getOpcode() == ISD::SHL_PARTS) { 4371 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4372 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4373 } else { 4374 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4375 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4376 } 4377 4378 SDOperand Ops[2] = { Lo, Hi }; 4379 return DAG.getMergeValues(Ops, 2); 4380} 4381 4382SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { 4383 MVT SrcVT = Op.getOperand(0).getValueType(); 4384 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4385 "Unknown SINT_TO_FP to lower!"); 4386 4387 // These are really Legal; caller falls through into that case. 4388 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4389 return SDOperand(); 4390 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4391 Subtarget->is64Bit()) 4392 return SDOperand(); 4393 4394 unsigned Size = SrcVT.getSizeInBits()/8; 4395 MachineFunction &MF = DAG.getMachineFunction(); 4396 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4397 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4398 SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4399 StackSlot, 4400 PseudoSourceValue::getFixedStack(SSFI), 0); 4401 4402 // Build the FILD 4403 SDVTList Tys; 4404 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4405 if (useSSE) 4406 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4407 else 4408 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4409 SmallVector<SDOperand, 8> Ops; 4410 Ops.push_back(Chain); 4411 Ops.push_back(StackSlot); 4412 Ops.push_back(DAG.getValueType(SrcVT)); 4413 SDOperand Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4414 Tys, &Ops[0], Ops.size()); 4415 4416 if (useSSE) { 4417 Chain = Result.getValue(1); 4418 SDOperand InFlag = Result.getValue(2); 4419 4420 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4421 // shouldn't be necessary except that RFP cannot be live across 4422 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4423 MachineFunction &MF = DAG.getMachineFunction(); 4424 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4425 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4426 Tys = DAG.getVTList(MVT::Other); 4427 SmallVector<SDOperand, 8> Ops; 4428 Ops.push_back(Chain); 4429 Ops.push_back(Result); 4430 Ops.push_back(StackSlot); 4431 Ops.push_back(DAG.getValueType(Op.getValueType())); 4432 Ops.push_back(InFlag); 4433 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4434 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4435 PseudoSourceValue::getFixedStack(SSFI), 0); 4436 } 4437 4438 return Result; 4439} 4440 4441std::pair<SDOperand,SDOperand> X86TargetLowering:: 4442FP_TO_SINTHelper(SDOperand Op, SelectionDAG &DAG) { 4443 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4444 Op.getValueType().getSimpleVT() >= MVT::i16 && 4445 "Unknown FP_TO_SINT to lower!"); 4446 4447 // These are really Legal. 4448 if (Op.getValueType() == MVT::i32 && 4449 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4450 return std::make_pair(SDOperand(), SDOperand()); 4451 if (Subtarget->is64Bit() && 4452 Op.getValueType() == MVT::i64 && 4453 Op.getOperand(0).getValueType() != MVT::f80) 4454 return std::make_pair(SDOperand(), SDOperand()); 4455 4456 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4457 // stack slot. 4458 MachineFunction &MF = DAG.getMachineFunction(); 4459 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4460 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4461 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4462 unsigned Opc; 4463 switch (Op.getValueType().getSimpleVT()) { 4464 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4465 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4466 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4467 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4468 } 4469 4470 SDOperand Chain = DAG.getEntryNode(); 4471 SDOperand Value = Op.getOperand(0); 4472 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4473 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4474 Chain = DAG.getStore(Chain, Value, StackSlot, 4475 PseudoSourceValue::getFixedStack(SSFI), 0); 4476 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4477 SDOperand Ops[] = { 4478 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4479 }; 4480 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4481 Chain = Value.getValue(1); 4482 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4483 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4484 } 4485 4486 // Build the FP_TO_INT*_IN_MEM 4487 SDOperand Ops[] = { Chain, Value, StackSlot }; 4488 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4489 4490 return std::make_pair(FIST, StackSlot); 4491} 4492 4493SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { 4494 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(Op, DAG); 4495 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4496 if (FIST.Val == 0) return SDOperand(); 4497 4498 // Load the result. 4499 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4500} 4501 4502SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4503 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(SDOperand(N, 0), DAG); 4504 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4505 if (FIST.Val == 0) return 0; 4506 4507 MVT VT = N->getValueType(0); 4508 4509 // Return a load from the stack slot. 4510 SDOperand Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0); 4511 4512 // Use MERGE_VALUES to drop the chain result value and get a node with one 4513 // result. This requires turning off getMergeValues simplification, since 4514 // otherwise it will give us Res back. 4515 return DAG.getMergeValues(&Res, 1, false).Val; 4516} 4517 4518SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) { 4519 MVT VT = Op.getValueType(); 4520 MVT EltVT = VT; 4521 if (VT.isVector()) 4522 EltVT = VT.getVectorElementType(); 4523 std::vector<Constant*> CV; 4524 if (EltVT == MVT::f64) { 4525 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4526 CV.push_back(C); 4527 CV.push_back(C); 4528 } else { 4529 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4530 CV.push_back(C); 4531 CV.push_back(C); 4532 CV.push_back(C); 4533 CV.push_back(C); 4534 } 4535 Constant *C = ConstantVector::get(CV); 4536 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4537 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4538 PseudoSourceValue::getConstantPool(), 0, 4539 false, 16); 4540 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4541} 4542 4543SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) { 4544 MVT VT = Op.getValueType(); 4545 MVT EltVT = VT; 4546 unsigned EltNum = 1; 4547 if (VT.isVector()) { 4548 EltVT = VT.getVectorElementType(); 4549 EltNum = VT.getVectorNumElements(); 4550 } 4551 std::vector<Constant*> CV; 4552 if (EltVT == MVT::f64) { 4553 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4554 CV.push_back(C); 4555 CV.push_back(C); 4556 } else { 4557 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4558 CV.push_back(C); 4559 CV.push_back(C); 4560 CV.push_back(C); 4561 CV.push_back(C); 4562 } 4563 Constant *C = ConstantVector::get(CV); 4564 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4565 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4566 PseudoSourceValue::getConstantPool(), 0, 4567 false, 16); 4568 if (VT.isVector()) { 4569 return DAG.getNode(ISD::BIT_CONVERT, VT, 4570 DAG.getNode(ISD::XOR, MVT::v2i64, 4571 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4572 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4573 } else { 4574 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4575 } 4576} 4577 4578SDOperand X86TargetLowering::LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) { 4579 SDOperand Op0 = Op.getOperand(0); 4580 SDOperand Op1 = Op.getOperand(1); 4581 MVT VT = Op.getValueType(); 4582 MVT SrcVT = Op1.getValueType(); 4583 4584 // If second operand is smaller, extend it first. 4585 if (SrcVT.bitsLT(VT)) { 4586 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4587 SrcVT = VT; 4588 } 4589 // And if it is bigger, shrink it first. 4590 if (SrcVT.bitsGT(VT)) { 4591 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4592 SrcVT = VT; 4593 } 4594 4595 // At this point the operands and the result should have the same 4596 // type, and that won't be f80 since that is not custom lowered. 4597 4598 // First get the sign bit of second operand. 4599 std::vector<Constant*> CV; 4600 if (SrcVT == MVT::f64) { 4601 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4602 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4603 } else { 4604 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4605 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4606 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4607 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4608 } 4609 Constant *C = ConstantVector::get(CV); 4610 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4611 SDOperand Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4612 PseudoSourceValue::getConstantPool(), 0, 4613 false, 16); 4614 SDOperand SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4615 4616 // Shift sign bit right or left if the two operands have different types. 4617 if (SrcVT.bitsGT(VT)) { 4618 // Op0 is MVT::f32, Op1 is MVT::f64. 4619 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4620 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4621 DAG.getConstant(32, MVT::i32)); 4622 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4623 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4624 DAG.getIntPtrConstant(0)); 4625 } 4626 4627 // Clear first operand sign bit. 4628 CV.clear(); 4629 if (VT == MVT::f64) { 4630 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4631 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4632 } else { 4633 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4634 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4635 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4636 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4637 } 4638 C = ConstantVector::get(CV); 4639 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4640 SDOperand Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4641 PseudoSourceValue::getConstantPool(), 0, 4642 false, 16); 4643 SDOperand Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4644 4645 // Or the value with the sign bit. 4646 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4647} 4648 4649SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG) { 4650 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4651 SDOperand Cond; 4652 SDOperand Op0 = Op.getOperand(0); 4653 SDOperand Op1 = Op.getOperand(1); 4654 SDOperand CC = Op.getOperand(2); 4655 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4656 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4657 unsigned X86CC; 4658 4659 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4660 Op0, Op1, DAG)) { 4661 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4662 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4663 DAG.getConstant(X86CC, MVT::i8), Cond); 4664 } 4665 4666 assert(isFP && "Illegal integer SetCC!"); 4667 4668 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4669 switch (SetCCOpcode) { 4670 default: assert(false && "Illegal floating point SetCC!"); 4671 case ISD::SETOEQ: { // !PF & ZF 4672 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4673 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4674 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4675 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4676 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4677 } 4678 case ISD::SETUNE: { // PF | !ZF 4679 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4680 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4681 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4682 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4683 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4684 } 4685 } 4686} 4687 4688 4689SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) { 4690 bool addTest = true; 4691 SDOperand Cond = Op.getOperand(0); 4692 SDOperand CC; 4693 4694 if (Cond.getOpcode() == ISD::SETCC) 4695 Cond = LowerSETCC(Cond, DAG); 4696 4697 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4698 // setting operand in place of the X86ISD::SETCC. 4699 if (Cond.getOpcode() == X86ISD::SETCC) { 4700 CC = Cond.getOperand(0); 4701 4702 SDOperand Cmp = Cond.getOperand(1); 4703 unsigned Opc = Cmp.getOpcode(); 4704 MVT VT = Op.getValueType(); 4705 4706 bool IllegalFPCMov = false; 4707 if (VT.isFloatingPoint() && !VT.isVector() && 4708 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4709 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4710 4711 if ((Opc == X86ISD::CMP || 4712 Opc == X86ISD::COMI || 4713 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4714 Cond = Cmp; 4715 addTest = false; 4716 } 4717 } 4718 4719 if (addTest) { 4720 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4721 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4722 } 4723 4724 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4725 MVT::Flag); 4726 SmallVector<SDOperand, 4> Ops; 4727 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4728 // condition is true. 4729 Ops.push_back(Op.getOperand(2)); 4730 Ops.push_back(Op.getOperand(1)); 4731 Ops.push_back(CC); 4732 Ops.push_back(Cond); 4733 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4734} 4735 4736SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) { 4737 bool addTest = true; 4738 SDOperand Chain = Op.getOperand(0); 4739 SDOperand Cond = Op.getOperand(1); 4740 SDOperand Dest = Op.getOperand(2); 4741 SDOperand CC; 4742 4743 if (Cond.getOpcode() == ISD::SETCC) 4744 Cond = LowerSETCC(Cond, DAG); 4745 4746 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4747 // setting operand in place of the X86ISD::SETCC. 4748 if (Cond.getOpcode() == X86ISD::SETCC) { 4749 CC = Cond.getOperand(0); 4750 4751 SDOperand Cmp = Cond.getOperand(1); 4752 unsigned Opc = Cmp.getOpcode(); 4753 if (Opc == X86ISD::CMP || 4754 Opc == X86ISD::COMI || 4755 Opc == X86ISD::UCOMI) { 4756 Cond = Cmp; 4757 addTest = false; 4758 } 4759 } 4760 4761 if (addTest) { 4762 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4763 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4764 } 4765 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 4766 Chain, Op.getOperand(2), CC, Cond); 4767} 4768 4769 4770// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 4771// Calls to _alloca is needed to probe the stack when allocating more than 4k 4772// bytes in one go. Touching the stack at 4K increments is necessary to ensure 4773// that the guard pages used by the OS virtual memory manager are allocated in 4774// correct sequence. 4775SDOperand 4776X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op, 4777 SelectionDAG &DAG) { 4778 assert(Subtarget->isTargetCygMing() && 4779 "This should be used only on Cygwin/Mingw targets"); 4780 4781 // Get the inputs. 4782 SDOperand Chain = Op.getOperand(0); 4783 SDOperand Size = Op.getOperand(1); 4784 // FIXME: Ensure alignment here 4785 4786 SDOperand Flag; 4787 4788 MVT IntPtr = getPointerTy(); 4789 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 4790 4791 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 4792 4793 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 4794 Flag = Chain.getValue(1); 4795 4796 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4797 SDOperand Ops[] = { Chain, 4798 DAG.getTargetExternalSymbol("_alloca", IntPtr), 4799 DAG.getRegister(X86::EAX, IntPtr), 4800 DAG.getRegister(X86StackPtr, SPTy), 4801 Flag }; 4802 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 4803 Flag = Chain.getValue(1); 4804 4805 Chain = DAG.getCALLSEQ_END(Chain, 4806 DAG.getIntPtrConstant(0), 4807 DAG.getIntPtrConstant(0), 4808 Flag); 4809 4810 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 4811 4812 SDOperand Ops1[2] = { Chain.getValue(0), Chain }; 4813 return DAG.getMergeValues(Ops1, 2); 4814} 4815 4816SDOperand 4817X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 4818 SDOperand Chain, 4819 SDOperand Dst, SDOperand Src, 4820 SDOperand Size, unsigned Align, 4821 const Value *DstSV, uint64_t DstSVOff) { 4822 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4823 4824 /// If not DWORD aligned or size is more than the threshold, call the library. 4825 /// The libc version is likely to be faster for these cases. It can use the 4826 /// address value and run time information about the CPU. 4827 if ((Align & 3) == 0 || 4828 !ConstantSize || 4829 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 4830 SDOperand InFlag(0, 0); 4831 4832 // Check to see if there is a specialized entry-point for memory zeroing. 4833 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 4834 if (const char *bzeroEntry = 4835 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 4836 MVT IntPtr = getPointerTy(); 4837 const Type *IntPtrTy = getTargetData()->getIntPtrType(); 4838 TargetLowering::ArgListTy Args; 4839 TargetLowering::ArgListEntry Entry; 4840 Entry.Node = Dst; 4841 Entry.Ty = IntPtrTy; 4842 Args.push_back(Entry); 4843 Entry.Node = Size; 4844 Args.push_back(Entry); 4845 std::pair<SDOperand,SDOperand> CallResult = 4846 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 4847 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 4848 Args, DAG); 4849 return CallResult.second; 4850 } 4851 4852 // Otherwise have the target-independent code call memset. 4853 return SDOperand(); 4854 } 4855 4856 uint64_t SizeVal = ConstantSize->getValue(); 4857 SDOperand InFlag(0, 0); 4858 MVT AVT; 4859 SDOperand Count; 4860 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 4861 unsigned BytesLeft = 0; 4862 bool TwoRepStos = false; 4863 if (ValC) { 4864 unsigned ValReg; 4865 uint64_t Val = ValC->getValue() & 255; 4866 4867 // If the value is a constant, then we can potentially use larger sets. 4868 switch (Align & 3) { 4869 case 2: // WORD aligned 4870 AVT = MVT::i16; 4871 ValReg = X86::AX; 4872 Val = (Val << 8) | Val; 4873 break; 4874 case 0: // DWORD aligned 4875 AVT = MVT::i32; 4876 ValReg = X86::EAX; 4877 Val = (Val << 8) | Val; 4878 Val = (Val << 16) | Val; 4879 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 4880 AVT = MVT::i64; 4881 ValReg = X86::RAX; 4882 Val = (Val << 32) | Val; 4883 } 4884 break; 4885 default: // Byte aligned 4886 AVT = MVT::i8; 4887 ValReg = X86::AL; 4888 Count = DAG.getIntPtrConstant(SizeVal); 4889 break; 4890 } 4891 4892 if (AVT.bitsGT(MVT::i8)) { 4893 unsigned UBytes = AVT.getSizeInBits() / 8; 4894 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 4895 BytesLeft = SizeVal % UBytes; 4896 } 4897 4898 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 4899 InFlag); 4900 InFlag = Chain.getValue(1); 4901 } else { 4902 AVT = MVT::i8; 4903 Count = DAG.getIntPtrConstant(SizeVal); 4904 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 4905 InFlag = Chain.getValue(1); 4906 } 4907 4908 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 4909 Count, InFlag); 4910 InFlag = Chain.getValue(1); 4911 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 4912 Dst, InFlag); 4913 InFlag = Chain.getValue(1); 4914 4915 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 4916 SmallVector<SDOperand, 8> Ops; 4917 Ops.push_back(Chain); 4918 Ops.push_back(DAG.getValueType(AVT)); 4919 Ops.push_back(InFlag); 4920 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 4921 4922 if (TwoRepStos) { 4923 InFlag = Chain.getValue(1); 4924 Count = Size; 4925 MVT CVT = Count.getValueType(); 4926 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count, 4927 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 4928 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 4929 Left, InFlag); 4930 InFlag = Chain.getValue(1); 4931 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 4932 Ops.clear(); 4933 Ops.push_back(Chain); 4934 Ops.push_back(DAG.getValueType(MVT::i8)); 4935 Ops.push_back(InFlag); 4936 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 4937 } else if (BytesLeft) { 4938 // Handle the last 1 - 7 bytes. 4939 unsigned Offset = SizeVal - BytesLeft; 4940 MVT AddrVT = Dst.getValueType(); 4941 MVT SizeVT = Size.getValueType(); 4942 4943 Chain = DAG.getMemset(Chain, 4944 DAG.getNode(ISD::ADD, AddrVT, Dst, 4945 DAG.getConstant(Offset, AddrVT)), 4946 Src, 4947 DAG.getConstant(BytesLeft, SizeVT), 4948 Align, DstSV, DstSVOff + Offset); 4949 } 4950 4951 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 4952 return Chain; 4953} 4954 4955SDOperand 4956X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 4957 SDOperand Chain, 4958 SDOperand Dst, SDOperand Src, 4959 SDOperand Size, unsigned Align, 4960 bool AlwaysInline, 4961 const Value *DstSV, uint64_t DstSVOff, 4962 const Value *SrcSV, uint64_t SrcSVOff){ 4963 4964 // This requires the copy size to be a constant, preferrably 4965 // within a subtarget-specific limit. 4966 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4967 if (!ConstantSize) 4968 return SDOperand(); 4969 uint64_t SizeVal = ConstantSize->getValue(); 4970 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 4971 return SDOperand(); 4972 4973 MVT AVT; 4974 unsigned BytesLeft = 0; 4975 if (Align >= 8 && Subtarget->is64Bit()) 4976 AVT = MVT::i64; 4977 else if (Align >= 4) 4978 AVT = MVT::i32; 4979 else if (Align >= 2) 4980 AVT = MVT::i16; 4981 else 4982 AVT = MVT::i8; 4983 4984 unsigned UBytes = AVT.getSizeInBits() / 8; 4985 unsigned CountVal = SizeVal / UBytes; 4986 SDOperand Count = DAG.getIntPtrConstant(CountVal); 4987 BytesLeft = SizeVal % UBytes; 4988 4989 SDOperand InFlag(0, 0); 4990 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 4991 Count, InFlag); 4992 InFlag = Chain.getValue(1); 4993 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 4994 Dst, InFlag); 4995 InFlag = Chain.getValue(1); 4996 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 4997 Src, InFlag); 4998 InFlag = Chain.getValue(1); 4999 5000 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5001 SmallVector<SDOperand, 8> Ops; 5002 Ops.push_back(Chain); 5003 Ops.push_back(DAG.getValueType(AVT)); 5004 Ops.push_back(InFlag); 5005 SDOperand RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5006 5007 SmallVector<SDOperand, 4> Results; 5008 Results.push_back(RepMovs); 5009 if (BytesLeft) { 5010 // Handle the last 1 - 7 bytes. 5011 unsigned Offset = SizeVal - BytesLeft; 5012 MVT DstVT = Dst.getValueType(); 5013 MVT SrcVT = Src.getValueType(); 5014 MVT SizeVT = Size.getValueType(); 5015 Results.push_back(DAG.getMemcpy(Chain, 5016 DAG.getNode(ISD::ADD, DstVT, Dst, 5017 DAG.getConstant(Offset, DstVT)), 5018 DAG.getNode(ISD::ADD, SrcVT, Src, 5019 DAG.getConstant(Offset, SrcVT)), 5020 DAG.getConstant(BytesLeft, SizeVT), 5021 Align, AlwaysInline, 5022 DstSV, DstSVOff + Offset, 5023 SrcSV, SrcSVOff + Offset)); 5024 } 5025 5026 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5027} 5028 5029/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5030SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5031 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5032 SDOperand TheChain = N->getOperand(0); 5033 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5034 if (Subtarget->is64Bit()) { 5035 SDOperand rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5036 SDOperand rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5037 MVT::i64, rax.getValue(2)); 5038 SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5039 DAG.getConstant(32, MVT::i8)); 5040 SDOperand Ops[] = { 5041 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5042 }; 5043 5044 return DAG.getMergeValues(Ops, 2).Val; 5045 } 5046 5047 SDOperand eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5048 SDOperand edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5049 MVT::i32, eax.getValue(2)); 5050 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5051 SDOperand Ops[] = { eax, edx }; 5052 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5053 5054 // Use a MERGE_VALUES to return the value and chain. 5055 Ops[1] = edx.getValue(1); 5056 return DAG.getMergeValues(Ops, 2).Val; 5057} 5058 5059SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) { 5060 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5061 5062 if (!Subtarget->is64Bit()) { 5063 // vastart just stores the address of the VarArgsFrameIndex slot into the 5064 // memory location argument. 5065 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5066 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5067 } 5068 5069 // __va_list_tag: 5070 // gp_offset (0 - 6 * 8) 5071 // fp_offset (48 - 48 + 8 * 16) 5072 // overflow_arg_area (point to parameters coming in memory). 5073 // reg_save_area 5074 SmallVector<SDOperand, 8> MemOps; 5075 SDOperand FIN = Op.getOperand(1); 5076 // Store gp_offset 5077 SDOperand Store = DAG.getStore(Op.getOperand(0), 5078 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5079 FIN, SV, 0); 5080 MemOps.push_back(Store); 5081 5082 // Store fp_offset 5083 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5084 Store = DAG.getStore(Op.getOperand(0), 5085 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5086 FIN, SV, 0); 5087 MemOps.push_back(Store); 5088 5089 // Store ptr to overflow_arg_area 5090 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5091 SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5092 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5093 MemOps.push_back(Store); 5094 5095 // Store ptr to reg_save_area. 5096 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5097 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5098 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5099 MemOps.push_back(Store); 5100 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5101} 5102 5103SDOperand X86TargetLowering::LowerVAARG(SDOperand Op, SelectionDAG &DAG) { 5104 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5105 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5106 SDOperand Chain = Op.getOperand(0); 5107 SDOperand SrcPtr = Op.getOperand(1); 5108 SDOperand SrcSV = Op.getOperand(2); 5109 5110 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5111 abort(); 5112 return SDOperand(); 5113} 5114 5115SDOperand X86TargetLowering::LowerVACOPY(SDOperand Op, SelectionDAG &DAG) { 5116 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5117 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5118 SDOperand Chain = Op.getOperand(0); 5119 SDOperand DstPtr = Op.getOperand(1); 5120 SDOperand SrcPtr = Op.getOperand(2); 5121 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5122 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5123 5124 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5125 DAG.getIntPtrConstant(24), 8, false, 5126 DstSV, 0, SrcSV, 0); 5127} 5128 5129SDOperand 5130X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) { 5131 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5132 switch (IntNo) { 5133 default: return SDOperand(); // Don't custom lower most intrinsics. 5134 // Comparison intrinsics. 5135 case Intrinsic::x86_sse_comieq_ss: 5136 case Intrinsic::x86_sse_comilt_ss: 5137 case Intrinsic::x86_sse_comile_ss: 5138 case Intrinsic::x86_sse_comigt_ss: 5139 case Intrinsic::x86_sse_comige_ss: 5140 case Intrinsic::x86_sse_comineq_ss: 5141 case Intrinsic::x86_sse_ucomieq_ss: 5142 case Intrinsic::x86_sse_ucomilt_ss: 5143 case Intrinsic::x86_sse_ucomile_ss: 5144 case Intrinsic::x86_sse_ucomigt_ss: 5145 case Intrinsic::x86_sse_ucomige_ss: 5146 case Intrinsic::x86_sse_ucomineq_ss: 5147 case Intrinsic::x86_sse2_comieq_sd: 5148 case Intrinsic::x86_sse2_comilt_sd: 5149 case Intrinsic::x86_sse2_comile_sd: 5150 case Intrinsic::x86_sse2_comigt_sd: 5151 case Intrinsic::x86_sse2_comige_sd: 5152 case Intrinsic::x86_sse2_comineq_sd: 5153 case Intrinsic::x86_sse2_ucomieq_sd: 5154 case Intrinsic::x86_sse2_ucomilt_sd: 5155 case Intrinsic::x86_sse2_ucomile_sd: 5156 case Intrinsic::x86_sse2_ucomigt_sd: 5157 case Intrinsic::x86_sse2_ucomige_sd: 5158 case Intrinsic::x86_sse2_ucomineq_sd: { 5159 unsigned Opc = 0; 5160 ISD::CondCode CC = ISD::SETCC_INVALID; 5161 switch (IntNo) { 5162 default: break; 5163 case Intrinsic::x86_sse_comieq_ss: 5164 case Intrinsic::x86_sse2_comieq_sd: 5165 Opc = X86ISD::COMI; 5166 CC = ISD::SETEQ; 5167 break; 5168 case Intrinsic::x86_sse_comilt_ss: 5169 case Intrinsic::x86_sse2_comilt_sd: 5170 Opc = X86ISD::COMI; 5171 CC = ISD::SETLT; 5172 break; 5173 case Intrinsic::x86_sse_comile_ss: 5174 case Intrinsic::x86_sse2_comile_sd: 5175 Opc = X86ISD::COMI; 5176 CC = ISD::SETLE; 5177 break; 5178 case Intrinsic::x86_sse_comigt_ss: 5179 case Intrinsic::x86_sse2_comigt_sd: 5180 Opc = X86ISD::COMI; 5181 CC = ISD::SETGT; 5182 break; 5183 case Intrinsic::x86_sse_comige_ss: 5184 case Intrinsic::x86_sse2_comige_sd: 5185 Opc = X86ISD::COMI; 5186 CC = ISD::SETGE; 5187 break; 5188 case Intrinsic::x86_sse_comineq_ss: 5189 case Intrinsic::x86_sse2_comineq_sd: 5190 Opc = X86ISD::COMI; 5191 CC = ISD::SETNE; 5192 break; 5193 case Intrinsic::x86_sse_ucomieq_ss: 5194 case Intrinsic::x86_sse2_ucomieq_sd: 5195 Opc = X86ISD::UCOMI; 5196 CC = ISD::SETEQ; 5197 break; 5198 case Intrinsic::x86_sse_ucomilt_ss: 5199 case Intrinsic::x86_sse2_ucomilt_sd: 5200 Opc = X86ISD::UCOMI; 5201 CC = ISD::SETLT; 5202 break; 5203 case Intrinsic::x86_sse_ucomile_ss: 5204 case Intrinsic::x86_sse2_ucomile_sd: 5205 Opc = X86ISD::UCOMI; 5206 CC = ISD::SETLE; 5207 break; 5208 case Intrinsic::x86_sse_ucomigt_ss: 5209 case Intrinsic::x86_sse2_ucomigt_sd: 5210 Opc = X86ISD::UCOMI; 5211 CC = ISD::SETGT; 5212 break; 5213 case Intrinsic::x86_sse_ucomige_ss: 5214 case Intrinsic::x86_sse2_ucomige_sd: 5215 Opc = X86ISD::UCOMI; 5216 CC = ISD::SETGE; 5217 break; 5218 case Intrinsic::x86_sse_ucomineq_ss: 5219 case Intrinsic::x86_sse2_ucomineq_sd: 5220 Opc = X86ISD::UCOMI; 5221 CC = ISD::SETNE; 5222 break; 5223 } 5224 5225 unsigned X86CC; 5226 SDOperand LHS = Op.getOperand(1); 5227 SDOperand RHS = Op.getOperand(2); 5228 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5229 5230 SDOperand Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5231 SDOperand SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5232 DAG.getConstant(X86CC, MVT::i8), Cond); 5233 return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC); 5234 } 5235 5236 // Fix vector shift instructions where the last operand is a non-immediate 5237 // i32 value. 5238 case Intrinsic::x86_sse2_pslli_w: 5239 case Intrinsic::x86_sse2_pslli_d: 5240 case Intrinsic::x86_sse2_pslli_q: 5241 case Intrinsic::x86_sse2_psrli_w: 5242 case Intrinsic::x86_sse2_psrli_d: 5243 case Intrinsic::x86_sse2_psrli_q: 5244 case Intrinsic::x86_sse2_psrai_w: 5245 case Intrinsic::x86_sse2_psrai_d: 5246 case Intrinsic::x86_mmx_pslli_w: 5247 case Intrinsic::x86_mmx_pslli_d: 5248 case Intrinsic::x86_mmx_pslli_q: 5249 case Intrinsic::x86_mmx_psrli_w: 5250 case Intrinsic::x86_mmx_psrli_d: 5251 case Intrinsic::x86_mmx_psrli_q: 5252 case Intrinsic::x86_mmx_psrai_w: 5253 case Intrinsic::x86_mmx_psrai_d: { 5254 SDOperand ShAmt = Op.getOperand(2); 5255 if (isa<ConstantSDNode>(ShAmt)) 5256 return SDOperand(); 5257 5258 unsigned NewIntNo = 0; 5259 MVT ShAmtVT = MVT::v4i32; 5260 switch (IntNo) { 5261 case Intrinsic::x86_sse2_pslli_w: 5262 NewIntNo = Intrinsic::x86_sse2_psll_w; 5263 break; 5264 case Intrinsic::x86_sse2_pslli_d: 5265 NewIntNo = Intrinsic::x86_sse2_psll_d; 5266 break; 5267 case Intrinsic::x86_sse2_pslli_q: 5268 NewIntNo = Intrinsic::x86_sse2_psll_q; 5269 break; 5270 case Intrinsic::x86_sse2_psrli_w: 5271 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5272 break; 5273 case Intrinsic::x86_sse2_psrli_d: 5274 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5275 break; 5276 case Intrinsic::x86_sse2_psrli_q: 5277 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5278 break; 5279 case Intrinsic::x86_sse2_psrai_w: 5280 NewIntNo = Intrinsic::x86_sse2_psra_w; 5281 break; 5282 case Intrinsic::x86_sse2_psrai_d: 5283 NewIntNo = Intrinsic::x86_sse2_psra_d; 5284 break; 5285 default: { 5286 ShAmtVT = MVT::v2i32; 5287 switch (IntNo) { 5288 case Intrinsic::x86_mmx_pslli_w: 5289 NewIntNo = Intrinsic::x86_mmx_psll_w; 5290 break; 5291 case Intrinsic::x86_mmx_pslli_d: 5292 NewIntNo = Intrinsic::x86_mmx_psll_d; 5293 break; 5294 case Intrinsic::x86_mmx_pslli_q: 5295 NewIntNo = Intrinsic::x86_mmx_psll_q; 5296 break; 5297 case Intrinsic::x86_mmx_psrli_w: 5298 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5299 break; 5300 case Intrinsic::x86_mmx_psrli_d: 5301 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5302 break; 5303 case Intrinsic::x86_mmx_psrli_q: 5304 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5305 break; 5306 case Intrinsic::x86_mmx_psrai_w: 5307 NewIntNo = Intrinsic::x86_mmx_psra_w; 5308 break; 5309 case Intrinsic::x86_mmx_psrai_d: 5310 NewIntNo = Intrinsic::x86_mmx_psra_d; 5311 break; 5312 default: abort(); // Can't reach here. 5313 } 5314 break; 5315 } 5316 } 5317 MVT VT = Op.getValueType(); 5318 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5319 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5320 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5321 DAG.getConstant(NewIntNo, MVT::i32), 5322 Op.getOperand(1), ShAmt); 5323 } 5324 } 5325} 5326 5327SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) { 5328 // Depths > 0 not supported yet! 5329 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5330 return SDOperand(); 5331 5332 // Just load the return address 5333 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5334 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5335} 5336 5337SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) { 5338 // Depths > 0 not supported yet! 5339 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5340 return SDOperand(); 5341 5342 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5343 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5344 DAG.getIntPtrConstant(!Subtarget->is64Bit() ? 4 : 8)); 5345} 5346 5347SDOperand X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDOperand Op, 5348 SelectionDAG &DAG) { 5349 // Is not yet supported on x86-64 5350 if (Subtarget->is64Bit()) 5351 return SDOperand(); 5352 5353 return DAG.getIntPtrConstant(8); 5354} 5355 5356SDOperand X86TargetLowering::LowerEH_RETURN(SDOperand Op, SelectionDAG &DAG) 5357{ 5358 assert(!Subtarget->is64Bit() && 5359 "Lowering of eh_return builtin is not supported yet on x86-64"); 5360 5361 MachineFunction &MF = DAG.getMachineFunction(); 5362 SDOperand Chain = Op.getOperand(0); 5363 SDOperand Offset = Op.getOperand(1); 5364 SDOperand Handler = Op.getOperand(2); 5365 5366 SDOperand Frame = DAG.getRegister(RegInfo->getFrameRegister(MF), 5367 getPointerTy()); 5368 5369 SDOperand StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5370 DAG.getIntPtrConstant(-4UL)); 5371 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5372 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5373 Chain = DAG.getCopyToReg(Chain, X86::ECX, StoreAddr); 5374 MF.getRegInfo().addLiveOut(X86::ECX); 5375 5376 return DAG.getNode(X86ISD::EH_RETURN, MVT::Other, 5377 Chain, DAG.getRegister(X86::ECX, getPointerTy())); 5378} 5379 5380SDOperand X86TargetLowering::LowerTRAMPOLINE(SDOperand Op, 5381 SelectionDAG &DAG) { 5382 SDOperand Root = Op.getOperand(0); 5383 SDOperand Trmp = Op.getOperand(1); // trampoline 5384 SDOperand FPtr = Op.getOperand(2); // nested function 5385 SDOperand Nest = Op.getOperand(3); // 'nest' parameter value 5386 5387 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5388 5389 const X86InstrInfo *TII = 5390 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5391 5392 if (Subtarget->is64Bit()) { 5393 SDOperand OutChains[6]; 5394 5395 // Large code-model. 5396 5397 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5398 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5399 5400 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5401 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5402 5403 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5404 5405 // Load the pointer to the nested function into R11. 5406 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5407 SDOperand Addr = Trmp; 5408 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5409 TrmpAddr, 0); 5410 5411 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5412 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5413 5414 // Load the 'nest' parameter value into R10. 5415 // R10 is specified in X86CallingConv.td 5416 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5417 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5418 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5419 TrmpAddr, 10); 5420 5421 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5422 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5423 5424 // Jump to the nested function. 5425 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5426 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5427 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5428 TrmpAddr, 20); 5429 5430 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5431 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5432 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5433 TrmpAddr, 22); 5434 5435 SDOperand Ops[] = 5436 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5437 return DAG.getMergeValues(Ops, 2); 5438 } else { 5439 const Function *Func = 5440 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5441 unsigned CC = Func->getCallingConv(); 5442 unsigned NestReg; 5443 5444 switch (CC) { 5445 default: 5446 assert(0 && "Unsupported calling convention"); 5447 case CallingConv::C: 5448 case CallingConv::X86_StdCall: { 5449 // Pass 'nest' parameter in ECX. 5450 // Must be kept in sync with X86CallingConv.td 5451 NestReg = X86::ECX; 5452 5453 // Check that ECX wasn't needed by an 'inreg' parameter. 5454 const FunctionType *FTy = Func->getFunctionType(); 5455 const PAListPtr &Attrs = Func->getParamAttrs(); 5456 5457 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5458 unsigned InRegCount = 0; 5459 unsigned Idx = 1; 5460 5461 for (FunctionType::param_iterator I = FTy->param_begin(), 5462 E = FTy->param_end(); I != E; ++I, ++Idx) 5463 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5464 // FIXME: should only count parameters that are lowered to integers. 5465 InRegCount += (getTargetData()->getTypeSizeInBits(*I) + 31) / 32; 5466 5467 if (InRegCount > 2) { 5468 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5469 abort(); 5470 } 5471 } 5472 break; 5473 } 5474 case CallingConv::X86_FastCall: 5475 // Pass 'nest' parameter in EAX. 5476 // Must be kept in sync with X86CallingConv.td 5477 NestReg = X86::EAX; 5478 break; 5479 } 5480 5481 SDOperand OutChains[4]; 5482 SDOperand Addr, Disp; 5483 5484 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5485 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5486 5487 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5488 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5489 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5490 Trmp, TrmpAddr, 0); 5491 5492 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5493 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5494 5495 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5496 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5497 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5498 TrmpAddr, 5, false, 1); 5499 5500 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5501 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5502 5503 SDOperand Ops[] = 5504 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5505 return DAG.getMergeValues(Ops, 2); 5506 } 5507} 5508 5509SDOperand X86TargetLowering::LowerFLT_ROUNDS_(SDOperand Op, SelectionDAG &DAG) { 5510 /* 5511 The rounding mode is in bits 11:10 of FPSR, and has the following 5512 settings: 5513 00 Round to nearest 5514 01 Round to -inf 5515 10 Round to +inf 5516 11 Round to 0 5517 5518 FLT_ROUNDS, on the other hand, expects the following: 5519 -1 Undefined 5520 0 Round to 0 5521 1 Round to nearest 5522 2 Round to +inf 5523 3 Round to -inf 5524 5525 To perform the conversion, we do: 5526 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5527 */ 5528 5529 MachineFunction &MF = DAG.getMachineFunction(); 5530 const TargetMachine &TM = MF.getTarget(); 5531 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5532 unsigned StackAlignment = TFI.getStackAlignment(); 5533 MVT VT = Op.getValueType(); 5534 5535 // Save FP Control Word to stack slot 5536 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5537 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5538 5539 SDOperand Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5540 DAG.getEntryNode(), StackSlot); 5541 5542 // Load FP Control Word from stack slot 5543 SDOperand CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5544 5545 // Transform as necessary 5546 SDOperand CWD1 = 5547 DAG.getNode(ISD::SRL, MVT::i16, 5548 DAG.getNode(ISD::AND, MVT::i16, 5549 CWD, DAG.getConstant(0x800, MVT::i16)), 5550 DAG.getConstant(11, MVT::i8)); 5551 SDOperand CWD2 = 5552 DAG.getNode(ISD::SRL, MVT::i16, 5553 DAG.getNode(ISD::AND, MVT::i16, 5554 CWD, DAG.getConstant(0x400, MVT::i16)), 5555 DAG.getConstant(9, MVT::i8)); 5556 5557 SDOperand RetVal = 5558 DAG.getNode(ISD::AND, MVT::i16, 5559 DAG.getNode(ISD::ADD, MVT::i16, 5560 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5561 DAG.getConstant(1, MVT::i16)), 5562 DAG.getConstant(3, MVT::i16)); 5563 5564 5565 return DAG.getNode((VT.getSizeInBits() < 16 ? 5566 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5567} 5568 5569SDOperand X86TargetLowering::LowerCTLZ(SDOperand Op, SelectionDAG &DAG) { 5570 MVT VT = Op.getValueType(); 5571 MVT OpVT = VT; 5572 unsigned NumBits = VT.getSizeInBits(); 5573 5574 Op = Op.getOperand(0); 5575 if (VT == MVT::i8) { 5576 // Zero extend to i32 since there is not an i8 bsr. 5577 OpVT = MVT::i32; 5578 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5579 } 5580 5581 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5582 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5583 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5584 5585 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5586 SmallVector<SDOperand, 4> Ops; 5587 Ops.push_back(Op); 5588 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5589 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5590 Ops.push_back(Op.getValue(1)); 5591 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5592 5593 // Finally xor with NumBits-1. 5594 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5595 5596 if (VT == MVT::i8) 5597 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5598 return Op; 5599} 5600 5601SDOperand X86TargetLowering::LowerCTTZ(SDOperand Op, SelectionDAG &DAG) { 5602 MVT VT = Op.getValueType(); 5603 MVT OpVT = VT; 5604 unsigned NumBits = VT.getSizeInBits(); 5605 5606 Op = Op.getOperand(0); 5607 if (VT == MVT::i8) { 5608 OpVT = MVT::i32; 5609 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5610 } 5611 5612 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5613 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5614 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5615 5616 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5617 SmallVector<SDOperand, 4> Ops; 5618 Ops.push_back(Op); 5619 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5620 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5621 Ops.push_back(Op.getValue(1)); 5622 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5623 5624 if (VT == MVT::i8) 5625 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5626 return Op; 5627} 5628 5629SDOperand X86TargetLowering::LowerCMP_SWAP(SDOperand Op, SelectionDAG &DAG) { 5630 MVT T = Op.getValueType(); 5631 unsigned Reg = 0; 5632 unsigned size = 0; 5633 switch(T.getSimpleVT()) { 5634 default: 5635 assert(false && "Invalid value type!"); 5636 case MVT::i8: Reg = X86::AL; size = 1; break; 5637 case MVT::i16: Reg = X86::AX; size = 2; break; 5638 case MVT::i32: Reg = X86::EAX; size = 4; break; 5639 case MVT::i64: 5640 if (Subtarget->is64Bit()) { 5641 Reg = X86::RAX; size = 8; 5642 } else //Should go away when LowerType stuff lands 5643 return SDOperand(ExpandATOMIC_CMP_SWAP(Op.Val, DAG), 0); 5644 break; 5645 }; 5646 SDOperand cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5647 Op.getOperand(3), SDOperand()); 5648 SDOperand Ops[] = { cpIn.getValue(0), 5649 Op.getOperand(1), 5650 Op.getOperand(2), 5651 DAG.getTargetConstant(size, MVT::i8), 5652 cpIn.getValue(1) }; 5653 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5654 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5655 SDOperand cpOut = 5656 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5657 return cpOut; 5658} 5659 5660SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, SelectionDAG &DAG) { 5661 MVT T = Op->getValueType(0); 5662 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5663 SDOperand cpInL, cpInH; 5664 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5665 DAG.getConstant(0, MVT::i32)); 5666 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5667 DAG.getConstant(1, MVT::i32)); 5668 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5669 cpInL, SDOperand()); 5670 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5671 cpInH, cpInL.getValue(1)); 5672 SDOperand swapInL, swapInH; 5673 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5674 DAG.getConstant(0, MVT::i32)); 5675 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5676 DAG.getConstant(1, MVT::i32)); 5677 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5678 swapInL, cpInH.getValue(1)); 5679 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5680 swapInH, swapInL.getValue(1)); 5681 SDOperand Ops[] = { swapInH.getValue(0), 5682 Op->getOperand(1), 5683 swapInH.getValue(1)}; 5684 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5685 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5686 SDOperand cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5687 Result.getValue(1)); 5688 SDOperand cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5689 cpOutL.getValue(2)); 5690 SDOperand OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5691 SDOperand ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5692 SDOperand Vals[2] = { ResultVal, cpOutH.getValue(1) }; 5693 return DAG.getMergeValues(Vals, 2).Val; 5694} 5695 5696SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, SelectionDAG &DAG) { 5697 MVT T = Op->getValueType(0); 5698 assert (T == MVT::i32 && "Only know how to expand i32 Atomic Load Sub"); 5699 SDOperand negOp = DAG.getNode(ISD::SUB, T, 5700 DAG.getConstant(0, T), Op->getOperand(2)); 5701 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, Op->getOperand(0), 5702 Op->getOperand(1), negOp, 5703 cast<AtomicSDNode>(Op)->getSrcValue(), 5704 cast<AtomicSDNode>(Op)->getAlignment()).Val; 5705} 5706 5707/// LowerOperation - Provide custom lowering hooks for some operations. 5708/// 5709SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 5710 switch (Op.getOpcode()) { 5711 default: assert(0 && "Should not custom lower this!"); 5712 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 5713 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5714 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5715 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5716 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5717 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5718 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5719 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5720 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5721 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5722 case ISD::SHL_PARTS: 5723 case ISD::SRA_PARTS: 5724 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5725 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5726 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5727 case ISD::FABS: return LowerFABS(Op, DAG); 5728 case ISD::FNEG: return LowerFNEG(Op, DAG); 5729 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5730 case ISD::SETCC: return LowerSETCC(Op, DAG); 5731 case ISD::SELECT: return LowerSELECT(Op, DAG); 5732 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5733 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5734 case ISD::CALL: return LowerCALL(Op, DAG); 5735 case ISD::RET: return LowerRET(Op, DAG); 5736 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 5737 case ISD::VASTART: return LowerVASTART(Op, DAG); 5738 case ISD::VAARG: return LowerVAARG(Op, DAG); 5739 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 5740 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 5741 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 5742 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 5743 case ISD::FRAME_TO_ARGS_OFFSET: 5744 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 5745 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 5746 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 5747 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 5748 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 5749 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 5750 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 5751 5752 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 5753 case ISD::READCYCLECOUNTER: 5754 return SDOperand(ExpandREADCYCLECOUNTER(Op.Val, DAG), 0); 5755 } 5756} 5757 5758/// ReplaceNodeResults - Replace a node with an illegal result type 5759/// with a new node built out of custom code. 5760SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) { 5761 switch (N->getOpcode()) { 5762 default: assert(0 && "Should not custom lower this!"); 5763 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 5764 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 5765 case ISD::ATOMIC_CMP_SWAP: return ExpandATOMIC_CMP_SWAP(N, DAG); 5766 case ISD::ATOMIC_LOAD_SUB: return ExpandATOMIC_LOAD_SUB(N,DAG); 5767 } 5768} 5769 5770const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 5771 switch (Opcode) { 5772 default: return NULL; 5773 case X86ISD::BSF: return "X86ISD::BSF"; 5774 case X86ISD::BSR: return "X86ISD::BSR"; 5775 case X86ISD::SHLD: return "X86ISD::SHLD"; 5776 case X86ISD::SHRD: return "X86ISD::SHRD"; 5777 case X86ISD::FAND: return "X86ISD::FAND"; 5778 case X86ISD::FOR: return "X86ISD::FOR"; 5779 case X86ISD::FXOR: return "X86ISD::FXOR"; 5780 case X86ISD::FSRL: return "X86ISD::FSRL"; 5781 case X86ISD::FILD: return "X86ISD::FILD"; 5782 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 5783 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 5784 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 5785 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 5786 case X86ISD::FLD: return "X86ISD::FLD"; 5787 case X86ISD::FST: return "X86ISD::FST"; 5788 case X86ISD::CALL: return "X86ISD::CALL"; 5789 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 5790 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 5791 case X86ISD::CMP: return "X86ISD::CMP"; 5792 case X86ISD::COMI: return "X86ISD::COMI"; 5793 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 5794 case X86ISD::SETCC: return "X86ISD::SETCC"; 5795 case X86ISD::CMOV: return "X86ISD::CMOV"; 5796 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 5797 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 5798 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 5799 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 5800 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 5801 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 5802 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 5803 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 5804 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 5805 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 5806 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 5807 case X86ISD::FMAX: return "X86ISD::FMAX"; 5808 case X86ISD::FMIN: return "X86ISD::FMIN"; 5809 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 5810 case X86ISD::FRCP: return "X86ISD::FRCP"; 5811 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 5812 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 5813 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 5814 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 5815 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 5816 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 5817 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 5818 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 5819 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 5820 case X86ISD::VSHL: return "X86ISD::VSHL"; 5821 case X86ISD::VSRL: return "X86ISD::VSRL"; 5822 } 5823} 5824 5825// isLegalAddressingMode - Return true if the addressing mode represented 5826// by AM is legal for this target, for a load/store of the specified type. 5827bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 5828 const Type *Ty) const { 5829 // X86 supports extremely general addressing modes. 5830 5831 // X86 allows a sign-extended 32-bit immediate field as a displacement. 5832 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 5833 return false; 5834 5835 if (AM.BaseGV) { 5836 // We can only fold this if we don't need an extra load. 5837 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 5838 return false; 5839 5840 // X86-64 only supports addr of globals in small code model. 5841 if (Subtarget->is64Bit()) { 5842 if (getTargetMachine().getCodeModel() != CodeModel::Small) 5843 return false; 5844 // If lower 4G is not available, then we must use rip-relative addressing. 5845 if (AM.BaseOffs || AM.Scale > 1) 5846 return false; 5847 } 5848 } 5849 5850 switch (AM.Scale) { 5851 case 0: 5852 case 1: 5853 case 2: 5854 case 4: 5855 case 8: 5856 // These scales always work. 5857 break; 5858 case 3: 5859 case 5: 5860 case 9: 5861 // These scales are formed with basereg+scalereg. Only accept if there is 5862 // no basereg yet. 5863 if (AM.HasBaseReg) 5864 return false; 5865 break; 5866 default: // Other stuff never works. 5867 return false; 5868 } 5869 5870 return true; 5871} 5872 5873 5874bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 5875 if (!Ty1->isInteger() || !Ty2->isInteger()) 5876 return false; 5877 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 5878 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 5879 if (NumBits1 <= NumBits2) 5880 return false; 5881 return Subtarget->is64Bit() || NumBits1 < 64; 5882} 5883 5884bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 5885 if (!VT1.isInteger() || !VT2.isInteger()) 5886 return false; 5887 unsigned NumBits1 = VT1.getSizeInBits(); 5888 unsigned NumBits2 = VT2.getSizeInBits(); 5889 if (NumBits1 <= NumBits2) 5890 return false; 5891 return Subtarget->is64Bit() || NumBits1 < 64; 5892} 5893 5894/// isShuffleMaskLegal - Targets can use this to indicate that they only 5895/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 5896/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 5897/// are assumed to be legal. 5898bool 5899X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT VT) const { 5900 // Only do shuffles on 128-bit vector types for now. 5901 if (VT.getSizeInBits() == 64) return false; 5902 return (Mask.Val->getNumOperands() <= 4 || 5903 isIdentityMask(Mask.Val) || 5904 isIdentityMask(Mask.Val, true) || 5905 isSplatMask(Mask.Val) || 5906 isPSHUFHW_PSHUFLWMask(Mask.Val) || 5907 X86::isUNPCKLMask(Mask.Val) || 5908 X86::isUNPCKHMask(Mask.Val) || 5909 X86::isUNPCKL_v_undef_Mask(Mask.Val) || 5910 X86::isUNPCKH_v_undef_Mask(Mask.Val)); 5911} 5912 5913bool 5914X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDOperand> &BVOps, 5915 MVT EVT, SelectionDAG &DAG) const { 5916 unsigned NumElts = BVOps.size(); 5917 // Only do shuffles on 128-bit vector types for now. 5918 if (EVT.getSizeInBits() * NumElts == 64) return false; 5919 if (NumElts == 2) return true; 5920 if (NumElts == 4) { 5921 return (isMOVLMask(&BVOps[0], 4) || 5922 isCommutedMOVL(&BVOps[0], 4, true) || 5923 isSHUFPMask(&BVOps[0], 4) || 5924 isCommutedSHUFP(&BVOps[0], 4)); 5925 } 5926 return false; 5927} 5928 5929//===----------------------------------------------------------------------===// 5930// X86 Scheduler Hooks 5931//===----------------------------------------------------------------------===// 5932 5933// private utility function 5934MachineBasicBlock * 5935X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 5936 MachineBasicBlock *MBB, 5937 unsigned regOpc, 5938 unsigned immOpc, 5939 bool invSrc) { 5940 // For the atomic bitwise operator, we generate 5941 // thisMBB: 5942 // newMBB: 5943 // ld t1 = [bitinstr.addr] 5944 // op t2 = t1, [bitinstr.val] 5945 // mov EAX = t1 5946 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 5947 // bz newMBB 5948 // fallthrough -->nextMBB 5949 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 5950 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 5951 MachineFunction::iterator MBBIter = MBB; 5952 ++MBBIter; 5953 5954 /// First build the CFG 5955 MachineFunction *F = MBB->getParent(); 5956 MachineBasicBlock *thisMBB = MBB; 5957 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 5958 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 5959 F->insert(MBBIter, newMBB); 5960 F->insert(MBBIter, nextMBB); 5961 5962 // Move all successors to thisMBB to nextMBB 5963 nextMBB->transferSuccessors(thisMBB); 5964 5965 // Update thisMBB to fall through to newMBB 5966 thisMBB->addSuccessor(newMBB); 5967 5968 // newMBB jumps to itself and fall through to nextMBB 5969 newMBB->addSuccessor(nextMBB); 5970 newMBB->addSuccessor(newMBB); 5971 5972 // Insert instructions into newMBB based on incoming instruction 5973 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 5974 MachineOperand& destOper = bInstr->getOperand(0); 5975 MachineOperand* argOpers[6]; 5976 int numArgs = bInstr->getNumOperands() - 1; 5977 for (int i=0; i < numArgs; ++i) 5978 argOpers[i] = &bInstr->getOperand(i+1); 5979 5980 // x86 address has 4 operands: base, index, scale, and displacement 5981 int lastAddrIndx = 3; // [0,3] 5982 int valArgIndx = 4; 5983 5984 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 5985 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 5986 for (int i=0; i <= lastAddrIndx; ++i) 5987 (*MIB).addOperand(*argOpers[i]); 5988 5989 unsigned tt = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 5990 if (invSrc) { 5991 MIB = BuildMI(newMBB, TII->get(X86::NOT32r), tt).addReg(t1); 5992 } 5993 else 5994 tt = t1; 5995 5996 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 5997 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 5998 && "invalid operand"); 5999 if (argOpers[valArgIndx]->isReg()) 6000 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6001 else 6002 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6003 MIB.addReg(tt); 6004 (*MIB).addOperand(*argOpers[valArgIndx]); 6005 6006 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6007 MIB.addReg(t1); 6008 6009 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6010 for (int i=0; i <= lastAddrIndx; ++i) 6011 (*MIB).addOperand(*argOpers[i]); 6012 MIB.addReg(t2); 6013 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6014 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 6015 6016 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6017 MIB.addReg(X86::EAX); 6018 6019 // insert branch 6020 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6021 6022 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 6023 return nextMBB; 6024} 6025 6026// private utility function 6027MachineBasicBlock * 6028X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6029 MachineBasicBlock *MBB, 6030 unsigned cmovOpc) { 6031 // For the atomic min/max operator, we generate 6032 // thisMBB: 6033 // newMBB: 6034 // ld t1 = [min/max.addr] 6035 // mov t2 = [min/max.val] 6036 // cmp t1, t2 6037 // cmov[cond] t2 = t1 6038 // mov EAX = t1 6039 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6040 // bz newMBB 6041 // fallthrough -->nextMBB 6042 // 6043 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6044 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6045 MachineFunction::iterator MBBIter = MBB; 6046 ++MBBIter; 6047 6048 /// First build the CFG 6049 MachineFunction *F = MBB->getParent(); 6050 MachineBasicBlock *thisMBB = MBB; 6051 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6052 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6053 F->insert(MBBIter, newMBB); 6054 F->insert(MBBIter, nextMBB); 6055 6056 // Move all successors to thisMBB to nextMBB 6057 nextMBB->transferSuccessors(thisMBB); 6058 6059 // Update thisMBB to fall through to newMBB 6060 thisMBB->addSuccessor(newMBB); 6061 6062 // newMBB jumps to newMBB and fall through to nextMBB 6063 newMBB->addSuccessor(nextMBB); 6064 newMBB->addSuccessor(newMBB); 6065 6066 // Insert instructions into newMBB based on incoming instruction 6067 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6068 MachineOperand& destOper = mInstr->getOperand(0); 6069 MachineOperand* argOpers[6]; 6070 int numArgs = mInstr->getNumOperands() - 1; 6071 for (int i=0; i < numArgs; ++i) 6072 argOpers[i] = &mInstr->getOperand(i+1); 6073 6074 // x86 address has 4 operands: base, index, scale, and displacement 6075 int lastAddrIndx = 3; // [0,3] 6076 int valArgIndx = 4; 6077 6078 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6079 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6080 for (int i=0; i <= lastAddrIndx; ++i) 6081 (*MIB).addOperand(*argOpers[i]); 6082 6083 // We only support register and immediate values 6084 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6085 && "invalid operand"); 6086 6087 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6088 if (argOpers[valArgIndx]->isReg()) 6089 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6090 else 6091 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6092 (*MIB).addOperand(*argOpers[valArgIndx]); 6093 6094 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6095 MIB.addReg(t1); 6096 6097 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6098 MIB.addReg(t1); 6099 MIB.addReg(t2); 6100 6101 // Generate movc 6102 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6103 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6104 MIB.addReg(t2); 6105 MIB.addReg(t1); 6106 6107 // Cmp and exchange if none has modified the memory location 6108 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6109 for (int i=0; i <= lastAddrIndx; ++i) 6110 (*MIB).addOperand(*argOpers[i]); 6111 MIB.addReg(t3); 6112 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6113 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 6114 6115 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6116 MIB.addReg(X86::EAX); 6117 6118 // insert branch 6119 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6120 6121 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 6122 return nextMBB; 6123} 6124 6125 6126MachineBasicBlock * 6127X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6128 MachineBasicBlock *BB) { 6129 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6130 switch (MI->getOpcode()) { 6131 default: assert(false && "Unexpected instr type to insert"); 6132 case X86::CMOV_FR32: 6133 case X86::CMOV_FR64: 6134 case X86::CMOV_V4F32: 6135 case X86::CMOV_V2F64: 6136 case X86::CMOV_V2I64: { 6137 // To "insert" a SELECT_CC instruction, we actually have to insert the 6138 // diamond control-flow pattern. The incoming instruction knows the 6139 // destination vreg to set, the condition code register to branch on, the 6140 // true/false values to select between, and a branch opcode to use. 6141 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6142 MachineFunction::iterator It = BB; 6143 ++It; 6144 6145 // thisMBB: 6146 // ... 6147 // TrueVal = ... 6148 // cmpTY ccX, r1, r2 6149 // bCC copy1MBB 6150 // fallthrough --> copy0MBB 6151 MachineBasicBlock *thisMBB = BB; 6152 MachineFunction *F = BB->getParent(); 6153 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6154 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6155 unsigned Opc = 6156 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6157 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6158 F->insert(It, copy0MBB); 6159 F->insert(It, sinkMBB); 6160 // Update machine-CFG edges by transferring all successors of the current 6161 // block to the new block which will contain the Phi node for the select. 6162 sinkMBB->transferSuccessors(BB); 6163 6164 // Add the true and fallthrough blocks as its successors. 6165 BB->addSuccessor(copy0MBB); 6166 BB->addSuccessor(sinkMBB); 6167 6168 // copy0MBB: 6169 // %FalseValue = ... 6170 // # fallthrough to sinkMBB 6171 BB = copy0MBB; 6172 6173 // Update machine-CFG edges 6174 BB->addSuccessor(sinkMBB); 6175 6176 // sinkMBB: 6177 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6178 // ... 6179 BB = sinkMBB; 6180 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6181 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6182 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6183 6184 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6185 return BB; 6186 } 6187 6188 case X86::FP32_TO_INT16_IN_MEM: 6189 case X86::FP32_TO_INT32_IN_MEM: 6190 case X86::FP32_TO_INT64_IN_MEM: 6191 case X86::FP64_TO_INT16_IN_MEM: 6192 case X86::FP64_TO_INT32_IN_MEM: 6193 case X86::FP64_TO_INT64_IN_MEM: 6194 case X86::FP80_TO_INT16_IN_MEM: 6195 case X86::FP80_TO_INT32_IN_MEM: 6196 case X86::FP80_TO_INT64_IN_MEM: { 6197 // Change the floating point control register to use "round towards zero" 6198 // mode when truncating to an integer value. 6199 MachineFunction *F = BB->getParent(); 6200 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6201 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6202 6203 // Load the old value of the high byte of the control word... 6204 unsigned OldCW = 6205 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6206 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6207 6208 // Set the high part to be round to zero... 6209 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6210 .addImm(0xC7F); 6211 6212 // Reload the modified control word now... 6213 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6214 6215 // Restore the memory image of control word to original value 6216 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6217 .addReg(OldCW); 6218 6219 // Get the X86 opcode to use. 6220 unsigned Opc; 6221 switch (MI->getOpcode()) { 6222 default: assert(0 && "illegal opcode!"); 6223 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6224 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6225 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6226 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6227 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6228 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6229 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6230 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6231 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6232 } 6233 6234 X86AddressMode AM; 6235 MachineOperand &Op = MI->getOperand(0); 6236 if (Op.isRegister()) { 6237 AM.BaseType = X86AddressMode::RegBase; 6238 AM.Base.Reg = Op.getReg(); 6239 } else { 6240 AM.BaseType = X86AddressMode::FrameIndexBase; 6241 AM.Base.FrameIndex = Op.getIndex(); 6242 } 6243 Op = MI->getOperand(1); 6244 if (Op.isImmediate()) 6245 AM.Scale = Op.getImm(); 6246 Op = MI->getOperand(2); 6247 if (Op.isImmediate()) 6248 AM.IndexReg = Op.getImm(); 6249 Op = MI->getOperand(3); 6250 if (Op.isGlobalAddress()) { 6251 AM.GV = Op.getGlobal(); 6252 } else { 6253 AM.Disp = Op.getImm(); 6254 } 6255 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6256 .addReg(MI->getOperand(4).getReg()); 6257 6258 // Reload the original control word now. 6259 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6260 6261 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6262 return BB; 6263 } 6264 case X86::ATOMAND32: 6265 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6266 X86::AND32ri); 6267 case X86::ATOMOR32: 6268 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6269 X86::OR32ri); 6270 case X86::ATOMXOR32: 6271 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6272 X86::XOR32ri); 6273 case X86::ATOMNAND32: 6274 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6275 X86::AND32ri, true); 6276 case X86::ATOMMIN32: 6277 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6278 case X86::ATOMMAX32: 6279 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6280 case X86::ATOMUMIN32: 6281 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6282 case X86::ATOMUMAX32: 6283 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6284 } 6285} 6286 6287//===----------------------------------------------------------------------===// 6288// X86 Optimization Hooks 6289//===----------------------------------------------------------------------===// 6290 6291void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op, 6292 const APInt &Mask, 6293 APInt &KnownZero, 6294 APInt &KnownOne, 6295 const SelectionDAG &DAG, 6296 unsigned Depth) const { 6297 unsigned Opc = Op.getOpcode(); 6298 assert((Opc >= ISD::BUILTIN_OP_END || 6299 Opc == ISD::INTRINSIC_WO_CHAIN || 6300 Opc == ISD::INTRINSIC_W_CHAIN || 6301 Opc == ISD::INTRINSIC_VOID) && 6302 "Should use MaskedValueIsZero if you don't know whether Op" 6303 " is a target node!"); 6304 6305 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6306 switch (Opc) { 6307 default: break; 6308 case X86ISD::SETCC: 6309 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6310 Mask.getBitWidth() - 1); 6311 break; 6312 } 6313} 6314 6315/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6316/// node is a GlobalAddress + offset. 6317bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6318 GlobalValue* &GA, int64_t &Offset) const{ 6319 if (N->getOpcode() == X86ISD::Wrapper) { 6320 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6321 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6322 return true; 6323 } 6324 } 6325 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6326} 6327 6328static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6329 const TargetLowering &TLI) { 6330 GlobalValue *GV; 6331 int64_t Offset = 0; 6332 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6333 return (GV->getAlignment() >= N && (Offset % N) == 0); 6334 // DAG combine handles the stack object case. 6335 return false; 6336} 6337 6338static bool EltsFromConsecutiveLoads(SDNode *N, SDOperand PermMask, 6339 unsigned NumElems, MVT EVT, 6340 SDNode *&Base, 6341 SelectionDAG &DAG, MachineFrameInfo *MFI, 6342 const TargetLowering &TLI) { 6343 Base = NULL; 6344 for (unsigned i = 0; i < NumElems; ++i) { 6345 SDOperand Idx = PermMask.getOperand(i); 6346 if (Idx.getOpcode() == ISD::UNDEF) { 6347 if (!Base) 6348 return false; 6349 continue; 6350 } 6351 6352 SDOperand Elt = DAG.getShuffleScalarElt(N, i); 6353 if (!Elt.Val || 6354 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.Val))) 6355 return false; 6356 if (!Base) { 6357 Base = Elt.Val; 6358 if (Base->getOpcode() == ISD::UNDEF) 6359 return false; 6360 continue; 6361 } 6362 if (Elt.getOpcode() == ISD::UNDEF) 6363 continue; 6364 6365 if (!TLI.isConsecutiveLoad(Elt.Val, Base, 6366 EVT.getSizeInBits()/8, i, MFI)) 6367 return false; 6368 } 6369 return true; 6370} 6371 6372/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6373/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6374/// if the load addresses are consecutive, non-overlapping, and in the right 6375/// order. 6376static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6377 const TargetLowering &TLI) { 6378 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6379 MVT VT = N->getValueType(0); 6380 MVT EVT = VT.getVectorElementType(); 6381 SDOperand PermMask = N->getOperand(2); 6382 unsigned NumElems = PermMask.getNumOperands(); 6383 SDNode *Base = NULL; 6384 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6385 DAG, MFI, TLI)) 6386 return SDOperand(); 6387 6388 LoadSDNode *LD = cast<LoadSDNode>(Base); 6389 if (isBaseAlignmentOfN(16, Base->getOperand(1).Val, TLI)) 6390 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6391 LD->getSrcValueOffset(), LD->isVolatile()); 6392 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6393 LD->getSrcValueOffset(), LD->isVolatile(), 6394 LD->getAlignment()); 6395} 6396 6397/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6398static SDOperand PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6399 const X86Subtarget *Subtarget, 6400 const TargetLowering &TLI) { 6401 unsigned NumOps = N->getNumOperands(); 6402 6403 // Ignore single operand BUILD_VECTOR. 6404 if (NumOps == 1) 6405 return SDOperand(); 6406 6407 MVT VT = N->getValueType(0); 6408 MVT EVT = VT.getVectorElementType(); 6409 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6410 // We are looking for load i64 and zero extend. We want to transform 6411 // it before legalizer has a chance to expand it. Also look for i64 6412 // BUILD_PAIR bit casted to f64. 6413 return SDOperand(); 6414 // This must be an insertion into a zero vector. 6415 SDOperand HighElt = N->getOperand(1); 6416 if (!isZeroNode(HighElt)) 6417 return SDOperand(); 6418 6419 // Value must be a load. 6420 SDNode *Base = N->getOperand(0).Val; 6421 if (!isa<LoadSDNode>(Base)) { 6422 if (Base->getOpcode() != ISD::BIT_CONVERT) 6423 return SDOperand(); 6424 Base = Base->getOperand(0).Val; 6425 if (!isa<LoadSDNode>(Base)) 6426 return SDOperand(); 6427 } 6428 6429 // Transform it into VZEXT_LOAD addr. 6430 LoadSDNode *LD = cast<LoadSDNode>(Base); 6431 6432 // Load must not be an extload. 6433 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6434 return SDOperand(); 6435 6436 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6437} 6438 6439/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6440static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6441 const X86Subtarget *Subtarget) { 6442 SDOperand Cond = N->getOperand(0); 6443 6444 // If we have SSE[12] support, try to form min/max nodes. 6445 if (Subtarget->hasSSE2() && 6446 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6447 if (Cond.getOpcode() == ISD::SETCC) { 6448 // Get the LHS/RHS of the select. 6449 SDOperand LHS = N->getOperand(1); 6450 SDOperand RHS = N->getOperand(2); 6451 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6452 6453 unsigned Opcode = 0; 6454 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6455 switch (CC) { 6456 default: break; 6457 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6458 case ISD::SETULE: 6459 case ISD::SETLE: 6460 if (!UnsafeFPMath) break; 6461 // FALL THROUGH. 6462 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6463 case ISD::SETLT: 6464 Opcode = X86ISD::FMIN; 6465 break; 6466 6467 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6468 case ISD::SETUGT: 6469 case ISD::SETGT: 6470 if (!UnsafeFPMath) break; 6471 // FALL THROUGH. 6472 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6473 case ISD::SETGE: 6474 Opcode = X86ISD::FMAX; 6475 break; 6476 } 6477 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6478 switch (CC) { 6479 default: break; 6480 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6481 case ISD::SETUGT: 6482 case ISD::SETGT: 6483 if (!UnsafeFPMath) break; 6484 // FALL THROUGH. 6485 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6486 case ISD::SETGE: 6487 Opcode = X86ISD::FMIN; 6488 break; 6489 6490 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6491 case ISD::SETULE: 6492 case ISD::SETLE: 6493 if (!UnsafeFPMath) break; 6494 // FALL THROUGH. 6495 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6496 case ISD::SETLT: 6497 Opcode = X86ISD::FMAX; 6498 break; 6499 } 6500 } 6501 6502 if (Opcode) 6503 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6504 } 6505 6506 } 6507 6508 return SDOperand(); 6509} 6510 6511/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6512static SDOperand PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6513 const X86Subtarget *Subtarget) { 6514 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6515 // the FP state in cases where an emms may be missing. 6516 // A preferable solution to the general problem is to figure out the right 6517 // places to insert EMMS. This qualifies as a quick hack. 6518 StoreSDNode *St = cast<StoreSDNode>(N); 6519 if (St->getValue().getValueType().isVector() && 6520 St->getValue().getValueType().getSizeInBits() == 64 && 6521 isa<LoadSDNode>(St->getValue()) && 6522 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6523 St->getChain().hasOneUse() && !St->isVolatile()) { 6524 SDNode* LdVal = St->getValue().Val; 6525 LoadSDNode *Ld = 0; 6526 int TokenFactorIndex = -1; 6527 SmallVector<SDOperand, 8> Ops; 6528 SDNode* ChainVal = St->getChain().Val; 6529 // Must be a store of a load. We currently handle two cases: the load 6530 // is a direct child, and it's under an intervening TokenFactor. It is 6531 // possible to dig deeper under nested TokenFactors. 6532 if (ChainVal == LdVal) 6533 Ld = cast<LoadSDNode>(St->getChain()); 6534 else if (St->getValue().hasOneUse() && 6535 ChainVal->getOpcode() == ISD::TokenFactor) { 6536 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6537 if (ChainVal->getOperand(i).Val == LdVal) { 6538 TokenFactorIndex = i; 6539 Ld = cast<LoadSDNode>(St->getValue()); 6540 } else 6541 Ops.push_back(ChainVal->getOperand(i)); 6542 } 6543 } 6544 if (Ld) { 6545 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6546 if (Subtarget->is64Bit()) { 6547 SDOperand NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6548 Ld->getBasePtr(), Ld->getSrcValue(), 6549 Ld->getSrcValueOffset(), Ld->isVolatile(), 6550 Ld->getAlignment()); 6551 SDOperand NewChain = NewLd.getValue(1); 6552 if (TokenFactorIndex != -1) { 6553 Ops.push_back(NewChain); 6554 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6555 Ops.size()); 6556 } 6557 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6558 St->getSrcValue(), St->getSrcValueOffset(), 6559 St->isVolatile(), St->getAlignment()); 6560 } 6561 6562 // Otherwise, lower to two 32-bit copies. 6563 SDOperand LoAddr = Ld->getBasePtr(); 6564 SDOperand HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6565 DAG.getConstant(4, MVT::i32)); 6566 6567 SDOperand LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6568 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6569 Ld->isVolatile(), Ld->getAlignment()); 6570 SDOperand HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6571 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6572 Ld->isVolatile(), 6573 MinAlign(Ld->getAlignment(), 4)); 6574 6575 SDOperand NewChain = LoLd.getValue(1); 6576 if (TokenFactorIndex != -1) { 6577 Ops.push_back(LoLd); 6578 Ops.push_back(HiLd); 6579 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6580 Ops.size()); 6581 } 6582 6583 LoAddr = St->getBasePtr(); 6584 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6585 DAG.getConstant(4, MVT::i32)); 6586 6587 SDOperand LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6588 St->getSrcValue(), St->getSrcValueOffset(), 6589 St->isVolatile(), St->getAlignment()); 6590 SDOperand HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6591 St->getSrcValue(), St->getSrcValueOffset()+4, 6592 St->isVolatile(), 6593 MinAlign(St->getAlignment(), 4)); 6594 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6595 } 6596 } 6597 return SDOperand(); 6598} 6599 6600/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6601/// X86ISD::FXOR nodes. 6602static SDOperand PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6603 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6604 // F[X]OR(0.0, x) -> x 6605 // F[X]OR(x, 0.0) -> x 6606 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6607 if (C->getValueAPF().isPosZero()) 6608 return N->getOperand(1); 6609 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6610 if (C->getValueAPF().isPosZero()) 6611 return N->getOperand(0); 6612 return SDOperand(); 6613} 6614 6615/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 6616static SDOperand PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 6617 // FAND(0.0, x) -> 0.0 6618 // FAND(x, 0.0) -> 0.0 6619 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6620 if (C->getValueAPF().isPosZero()) 6621 return N->getOperand(0); 6622 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6623 if (C->getValueAPF().isPosZero()) 6624 return N->getOperand(1); 6625 return SDOperand(); 6626} 6627 6628 6629SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N, 6630 DAGCombinerInfo &DCI) const { 6631 SelectionDAG &DAG = DCI.DAG; 6632 switch (N->getOpcode()) { 6633 default: break; 6634 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 6635 case ISD::BUILD_VECTOR: 6636 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 6637 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 6638 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 6639 case X86ISD::FXOR: 6640 case X86ISD::FOR: return PerformFORCombine(N, DAG); 6641 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 6642 } 6643 6644 return SDOperand(); 6645} 6646 6647//===----------------------------------------------------------------------===// 6648// X86 Inline Assembly Support 6649//===----------------------------------------------------------------------===// 6650 6651/// getConstraintType - Given a constraint letter, return the type of 6652/// constraint it is for this target. 6653X86TargetLowering::ConstraintType 6654X86TargetLowering::getConstraintType(const std::string &Constraint) const { 6655 if (Constraint.size() == 1) { 6656 switch (Constraint[0]) { 6657 case 'A': 6658 case 'f': 6659 case 'r': 6660 case 'R': 6661 case 'l': 6662 case 'q': 6663 case 'Q': 6664 case 'x': 6665 case 'y': 6666 case 'Y': 6667 return C_RegisterClass; 6668 default: 6669 break; 6670 } 6671 } 6672 return TargetLowering::getConstraintType(Constraint); 6673} 6674 6675/// LowerXConstraint - try to replace an X constraint, which matches anything, 6676/// with another that has more specific requirements based on the type of the 6677/// corresponding operand. 6678const char *X86TargetLowering:: 6679LowerXConstraint(MVT ConstraintVT) const { 6680 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 6681 // 'f' like normal targets. 6682 if (ConstraintVT.isFloatingPoint()) { 6683 if (Subtarget->hasSSE2()) 6684 return "Y"; 6685 if (Subtarget->hasSSE1()) 6686 return "x"; 6687 } 6688 6689 return TargetLowering::LowerXConstraint(ConstraintVT); 6690} 6691 6692/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 6693/// vector. If it is invalid, don't add anything to Ops. 6694void X86TargetLowering::LowerAsmOperandForConstraint(SDOperand Op, 6695 char Constraint, 6696 std::vector<SDOperand>&Ops, 6697 SelectionDAG &DAG) const { 6698 SDOperand Result(0, 0); 6699 6700 switch (Constraint) { 6701 default: break; 6702 case 'I': 6703 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6704 if (C->getValue() <= 31) { 6705 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6706 break; 6707 } 6708 } 6709 return; 6710 case 'N': 6711 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6712 if (C->getValue() <= 255) { 6713 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6714 break; 6715 } 6716 } 6717 return; 6718 case 'i': { 6719 // Literal immediates are always ok. 6720 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 6721 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 6722 break; 6723 } 6724 6725 // If we are in non-pic codegen mode, we allow the address of a global (with 6726 // an optional displacement) to be used with 'i'. 6727 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 6728 int64_t Offset = 0; 6729 6730 // Match either (GA) or (GA+C) 6731 if (GA) { 6732 Offset = GA->getOffset(); 6733 } else if (Op.getOpcode() == ISD::ADD) { 6734 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6735 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6736 if (C && GA) { 6737 Offset = GA->getOffset()+C->getValue(); 6738 } else { 6739 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6740 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6741 if (C && GA) 6742 Offset = GA->getOffset()+C->getValue(); 6743 else 6744 C = 0, GA = 0; 6745 } 6746 } 6747 6748 if (GA) { 6749 // If addressing this global requires a load (e.g. in PIC mode), we can't 6750 // match. 6751 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 6752 false)) 6753 return; 6754 6755 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 6756 Offset); 6757 Result = Op; 6758 break; 6759 } 6760 6761 // Otherwise, not valid for this mode. 6762 return; 6763 } 6764 } 6765 6766 if (Result.Val) { 6767 Ops.push_back(Result); 6768 return; 6769 } 6770 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 6771} 6772 6773std::vector<unsigned> X86TargetLowering:: 6774getRegClassForInlineAsmConstraint(const std::string &Constraint, 6775 MVT VT) const { 6776 if (Constraint.size() == 1) { 6777 // FIXME: not handling fp-stack yet! 6778 switch (Constraint[0]) { // GCC X86 Constraint Letters 6779 default: break; // Unknown constraint letter 6780 case 'A': // EAX/EDX 6781 if (VT == MVT::i32 || VT == MVT::i64) 6782 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 6783 break; 6784 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 6785 case 'Q': // Q_REGS 6786 if (VT == MVT::i32) 6787 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 6788 else if (VT == MVT::i16) 6789 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 6790 else if (VT == MVT::i8) 6791 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 6792 else if (VT == MVT::i64) 6793 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 6794 break; 6795 } 6796 } 6797 6798 return std::vector<unsigned>(); 6799} 6800 6801std::pair<unsigned, const TargetRegisterClass*> 6802X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 6803 MVT VT) const { 6804 // First, see if this is a constraint that directly corresponds to an LLVM 6805 // register class. 6806 if (Constraint.size() == 1) { 6807 // GCC Constraint Letters 6808 switch (Constraint[0]) { 6809 default: break; 6810 case 'r': // GENERAL_REGS 6811 case 'R': // LEGACY_REGS 6812 case 'l': // INDEX_REGS 6813 if (VT == MVT::i64 && Subtarget->is64Bit()) 6814 return std::make_pair(0U, X86::GR64RegisterClass); 6815 if (VT == MVT::i32) 6816 return std::make_pair(0U, X86::GR32RegisterClass); 6817 else if (VT == MVT::i16) 6818 return std::make_pair(0U, X86::GR16RegisterClass); 6819 else if (VT == MVT::i8) 6820 return std::make_pair(0U, X86::GR8RegisterClass); 6821 break; 6822 case 'f': // FP Stack registers. 6823 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 6824 // value to the correct fpstack register class. 6825 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 6826 return std::make_pair(0U, X86::RFP32RegisterClass); 6827 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 6828 return std::make_pair(0U, X86::RFP64RegisterClass); 6829 return std::make_pair(0U, X86::RFP80RegisterClass); 6830 case 'y': // MMX_REGS if MMX allowed. 6831 if (!Subtarget->hasMMX()) break; 6832 return std::make_pair(0U, X86::VR64RegisterClass); 6833 break; 6834 case 'Y': // SSE_REGS if SSE2 allowed 6835 if (!Subtarget->hasSSE2()) break; 6836 // FALL THROUGH. 6837 case 'x': // SSE_REGS if SSE1 allowed 6838 if (!Subtarget->hasSSE1()) break; 6839 6840 switch (VT.getSimpleVT()) { 6841 default: break; 6842 // Scalar SSE types. 6843 case MVT::f32: 6844 case MVT::i32: 6845 return std::make_pair(0U, X86::FR32RegisterClass); 6846 case MVT::f64: 6847 case MVT::i64: 6848 return std::make_pair(0U, X86::FR64RegisterClass); 6849 // Vector types. 6850 case MVT::v16i8: 6851 case MVT::v8i16: 6852 case MVT::v4i32: 6853 case MVT::v2i64: 6854 case MVT::v4f32: 6855 case MVT::v2f64: 6856 return std::make_pair(0U, X86::VR128RegisterClass); 6857 } 6858 break; 6859 } 6860 } 6861 6862 // Use the default implementation in TargetLowering to convert the register 6863 // constraint into a member of a register class. 6864 std::pair<unsigned, const TargetRegisterClass*> Res; 6865 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 6866 6867 // Not found as a standard register? 6868 if (Res.second == 0) { 6869 // GCC calls "st(0)" just plain "st". 6870 if (StringsEqualNoCase("{st}", Constraint)) { 6871 Res.first = X86::ST0; 6872 Res.second = X86::RFP80RegisterClass; 6873 } 6874 6875 return Res; 6876 } 6877 6878 // Otherwise, check to see if this is a register class of the wrong value 6879 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 6880 // turn into {ax},{dx}. 6881 if (Res.second->hasType(VT)) 6882 return Res; // Correct type already, nothing to do. 6883 6884 // All of the single-register GCC register classes map their values onto 6885 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 6886 // really want an 8-bit or 32-bit register, map to the appropriate register 6887 // class and return the appropriate register. 6888 if (Res.second != X86::GR16RegisterClass) 6889 return Res; 6890 6891 if (VT == MVT::i8) { 6892 unsigned DestReg = 0; 6893 switch (Res.first) { 6894 default: break; 6895 case X86::AX: DestReg = X86::AL; break; 6896 case X86::DX: DestReg = X86::DL; break; 6897 case X86::CX: DestReg = X86::CL; break; 6898 case X86::BX: DestReg = X86::BL; break; 6899 } 6900 if (DestReg) { 6901 Res.first = DestReg; 6902 Res.second = Res.second = X86::GR8RegisterClass; 6903 } 6904 } else if (VT == MVT::i32) { 6905 unsigned DestReg = 0; 6906 switch (Res.first) { 6907 default: break; 6908 case X86::AX: DestReg = X86::EAX; break; 6909 case X86::DX: DestReg = X86::EDX; break; 6910 case X86::CX: DestReg = X86::ECX; break; 6911 case X86::BX: DestReg = X86::EBX; break; 6912 case X86::SI: DestReg = X86::ESI; break; 6913 case X86::DI: DestReg = X86::EDI; break; 6914 case X86::BP: DestReg = X86::EBP; break; 6915 case X86::SP: DestReg = X86::ESP; break; 6916 } 6917 if (DestReg) { 6918 Res.first = DestReg; 6919 Res.second = Res.second = X86::GR32RegisterClass; 6920 } 6921 } else if (VT == MVT::i64) { 6922 unsigned DestReg = 0; 6923 switch (Res.first) { 6924 default: break; 6925 case X86::AX: DestReg = X86::RAX; break; 6926 case X86::DX: DestReg = X86::RDX; break; 6927 case X86::CX: DestReg = X86::RCX; break; 6928 case X86::BX: DestReg = X86::RBX; break; 6929 case X86::SI: DestReg = X86::RSI; break; 6930 case X86::DI: DestReg = X86::RDI; break; 6931 case X86::BP: DestReg = X86::RBP; break; 6932 case X86::SP: DestReg = X86::RSP; break; 6933 } 6934 if (DestReg) { 6935 Res.first = DestReg; 6936 Res.second = Res.second = X86::GR64RegisterClass; 6937 } 6938 } 6939 6940 return Res; 6941} 6942