X86ISelLowering.cpp revision 28873106309db515d58889a4c4fa3e0a92d1b60e
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86TargetMachine.h" 20#include "llvm/CallingConv.h" 21#include "llvm/Constants.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/GlobalVariable.h" 24#include "llvm/Function.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/ADT/BitVector.h" 27#include "llvm/ADT/VectorExtras.h" 28#include "llvm/CodeGen/CallingConvLower.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/PseudoSourceValue.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/Debug.h" 38#include "llvm/Target/TargetOptions.h" 39#include "llvm/ADT/SmallSet.h" 40#include "llvm/ADT/StringExtras.h" 41using namespace llvm; 42 43// Forward declarations. 44static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45 46X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47 : TargetLowering(TM) { 48 Subtarget = &TM.getSubtarget<X86Subtarget>(); 49 X86ScalarSSEf64 = Subtarget->hasSSE2(); 50 X86ScalarSSEf32 = Subtarget->hasSSE1(); 51 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52 53 bool Fast = false; 54 55 RegInfo = TM.getRegisterInfo(); 56 57 // Set up the TargetLowering object. 58 59 // X86 is weird, it always uses i8 for shift amounts and setcc results. 60 setShiftAmountType(MVT::i8); 61 setSetCCResultContents(ZeroOrOneSetCCResult); 62 setSchedulingPreference(SchedulingForRegPressure); 63 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 64 setStackPointerRegisterToSaveRestore(X86StackPtr); 65 66 if (Subtarget->isTargetDarwin()) { 67 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 68 setUseUnderscoreSetJmp(false); 69 setUseUnderscoreLongJmp(false); 70 } else if (Subtarget->isTargetMingw()) { 71 // MS runtime is weird: it exports _setjmp, but longjmp! 72 setUseUnderscoreSetJmp(true); 73 setUseUnderscoreLongJmp(false); 74 } else { 75 setUseUnderscoreSetJmp(true); 76 setUseUnderscoreLongJmp(true); 77 } 78 79 // Set up the register classes. 80 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 81 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 82 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 83 if (Subtarget->is64Bit()) 84 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 85 86 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 87 88 // We don't accept any truncstore of integer registers. 89 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 90 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 91 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 92 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 93 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 94 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 95 96 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 97 // operation. 98 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 99 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 100 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 101 102 if (Subtarget->is64Bit()) { 103 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 104 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 105 } else { 106 if (X86ScalarSSEf64) 107 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 108 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 109 else 110 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 111 } 112 113 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 114 // this operation. 115 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 116 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 117 // SSE has no i16 to fp conversion, only i32 118 if (X86ScalarSSEf32) { 119 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 120 // f32 and f64 cases are Legal, f80 case is not 121 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 122 } else { 123 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 124 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 125 } 126 127 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 128 // are Legal, f80 is custom lowered. 129 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 130 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 131 132 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 133 // this operation. 134 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 135 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 136 137 if (X86ScalarSSEf32) { 138 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 139 // f32 and f64 cases are Legal, f80 case is not 140 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 141 } else { 142 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 143 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 144 } 145 146 // Handle FP_TO_UINT by promoting the destination to a larger signed 147 // conversion. 148 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 149 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 151 152 if (Subtarget->is64Bit()) { 153 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 155 } else { 156 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 157 // Expand FP_TO_UINT into a select. 158 // FIXME: We would like to use a Custom expander here eventually to do 159 // the optimal thing for SSE vs. the default expansion in the legalizer. 160 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 161 else 162 // With SSE3 we can use fisttpll to convert to a signed i64. 163 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 164 } 165 166 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 167 if (!X86ScalarSSEf64) { 168 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 169 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 170 } 171 172 // Scalar integer divide and remainder are lowered to use operations that 173 // produce two results, to match the available instructions. This exposes 174 // the two-result form to trivial CSE, which is able to combine x/y and x%y 175 // into a single instruction. 176 // 177 // Scalar integer multiply-high is also lowered to use two-result 178 // operations, to match the available instructions. However, plain multiply 179 // (low) operations are left as Legal, as there are single-result 180 // instructions for this in x86. Using the two-result multiply instructions 181 // when both high and low results are needed must be arranged by dagcombine. 182 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 183 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 184 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 185 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::SREM , MVT::i8 , Expand); 187 setOperationAction(ISD::UREM , MVT::i8 , Expand); 188 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 189 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 190 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 191 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::SREM , MVT::i16 , Expand); 193 setOperationAction(ISD::UREM , MVT::i16 , Expand); 194 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 195 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 196 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 197 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::SREM , MVT::i32 , Expand); 199 setOperationAction(ISD::UREM , MVT::i32 , Expand); 200 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 201 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 202 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 203 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::SREM , MVT::i64 , Expand); 205 setOperationAction(ISD::UREM , MVT::i64 , Expand); 206 207 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 208 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 209 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 210 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 211 if (Subtarget->is64Bit()) 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 216 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 217 setOperationAction(ISD::FREM , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f64 , Expand); 219 setOperationAction(ISD::FREM , MVT::f80 , Expand); 220 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 221 222 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 223 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 224 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 226 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 227 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 229 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 230 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 231 if (Subtarget->is64Bit()) { 232 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 233 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 234 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 235 } 236 237 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 238 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 239 240 // These should be promoted to a larger select which is supported. 241 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 242 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 243 // X86 wants to expand cmov itself. 244 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 245 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 246 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 249 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 252 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 255 if (Subtarget->is64Bit()) { 256 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 257 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 258 } 259 // X86 ret instruction may pop stack. 260 setOperationAction(ISD::RET , MVT::Other, Custom); 261 if (!Subtarget->is64Bit()) 262 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 263 264 // Darwin ABI issue. 265 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 266 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 268 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 269 if (Subtarget->is64Bit()) 270 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 271 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 272 if (Subtarget->is64Bit()) { 273 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 274 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 275 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 276 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 277 } 278 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 279 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 281 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 282 if (Subtarget->is64Bit()) { 283 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 285 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 286 } 287 288 if (Subtarget->hasSSE1()) 289 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 290 291 if (!Subtarget->hasSSE2()) 292 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 293 294 // Expand certain atomics 295 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i8, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i16, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i32, Custom); 298 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i64, Custom); 299 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i32, Expand); 300 301 // Use the default ISD::LOCATION, ISD::DECLARE expansion. 302 setOperationAction(ISD::LOCATION, MVT::Other, Expand); 303 // FIXME - use subtarget debug flags 304 if (!Subtarget->isTargetDarwin() && 305 !Subtarget->isTargetELF() && 306 !Subtarget->isTargetCygMing()) 307 setOperationAction(ISD::LABEL, MVT::Other, Expand); 308 309 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 310 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 311 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 312 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 313 if (Subtarget->is64Bit()) { 314 // FIXME: Verify 315 setExceptionPointerRegister(X86::RAX); 316 setExceptionSelectorRegister(X86::RDX); 317 } else { 318 setExceptionPointerRegister(X86::EAX); 319 setExceptionSelectorRegister(X86::EDX); 320 } 321 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 322 323 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 324 325 setOperationAction(ISD::TRAP, MVT::Other, Legal); 326 327 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 328 setOperationAction(ISD::VASTART , MVT::Other, Custom); 329 setOperationAction(ISD::VAEND , MVT::Other, Expand); 330 if (Subtarget->is64Bit()) { 331 setOperationAction(ISD::VAARG , MVT::Other, Custom); 332 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 333 } else { 334 setOperationAction(ISD::VAARG , MVT::Other, Expand); 335 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 336 } 337 338 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 339 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 340 if (Subtarget->is64Bit()) 341 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 342 if (Subtarget->isTargetCygMing()) 343 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 344 else 345 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 346 347 if (X86ScalarSSEf64) { 348 // f32 and f64 use SSE. 349 // Set up the FP register classes. 350 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 351 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 352 353 // Use ANDPD to simulate FABS. 354 setOperationAction(ISD::FABS , MVT::f64, Custom); 355 setOperationAction(ISD::FABS , MVT::f32, Custom); 356 357 // Use XORP to simulate FNEG. 358 setOperationAction(ISD::FNEG , MVT::f64, Custom); 359 setOperationAction(ISD::FNEG , MVT::f32, Custom); 360 361 // Use ANDPD and ORPD to simulate FCOPYSIGN. 362 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 363 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 364 365 // We don't support sin/cos/fmod 366 setOperationAction(ISD::FSIN , MVT::f64, Expand); 367 setOperationAction(ISD::FCOS , MVT::f64, Expand); 368 setOperationAction(ISD::FSIN , MVT::f32, Expand); 369 setOperationAction(ISD::FCOS , MVT::f32, Expand); 370 371 // Expand FP immediates into loads from the stack, except for the special 372 // cases we handle. 373 addLegalFPImmediate(APFloat(+0.0)); // xorpd 374 addLegalFPImmediate(APFloat(+0.0f)); // xorps 375 376 // Floating truncations from f80 and extensions to f80 go through memory. 377 // If optimizing, we lie about this though and handle it in 378 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 379 if (Fast) { 380 setConvertAction(MVT::f32, MVT::f80, Expand); 381 setConvertAction(MVT::f64, MVT::f80, Expand); 382 setConvertAction(MVT::f80, MVT::f32, Expand); 383 setConvertAction(MVT::f80, MVT::f64, Expand); 384 } 385 } else if (X86ScalarSSEf32) { 386 // Use SSE for f32, x87 for f64. 387 // Set up the FP register classes. 388 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 389 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 390 391 // Use ANDPS to simulate FABS. 392 setOperationAction(ISD::FABS , MVT::f32, Custom); 393 394 // Use XORP to simulate FNEG. 395 setOperationAction(ISD::FNEG , MVT::f32, Custom); 396 397 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 398 399 // Use ANDPS and ORPS to simulate FCOPYSIGN. 400 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 401 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 402 403 // We don't support sin/cos/fmod 404 setOperationAction(ISD::FSIN , MVT::f32, Expand); 405 setOperationAction(ISD::FCOS , MVT::f32, Expand); 406 407 // Special cases we handle for FP constants. 408 addLegalFPImmediate(APFloat(+0.0f)); // xorps 409 addLegalFPImmediate(APFloat(+0.0)); // FLD0 410 addLegalFPImmediate(APFloat(+1.0)); // FLD1 411 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 412 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 413 414 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 415 // this though and handle it in InstructionSelectPreprocess so that 416 // dagcombine2 can hack on these. 417 if (Fast) { 418 setConvertAction(MVT::f32, MVT::f64, Expand); 419 setConvertAction(MVT::f32, MVT::f80, Expand); 420 setConvertAction(MVT::f80, MVT::f32, Expand); 421 setConvertAction(MVT::f64, MVT::f32, Expand); 422 // And x87->x87 truncations also. 423 setConvertAction(MVT::f80, MVT::f64, Expand); 424 } 425 426 if (!UnsafeFPMath) { 427 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 428 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 429 } 430 } else { 431 // f32 and f64 in x87. 432 // Set up the FP register classes. 433 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 434 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 435 436 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 437 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 438 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 439 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 440 441 // Floating truncations go through memory. If optimizing, we lie about 442 // this though and handle it in InstructionSelectPreprocess so that 443 // dagcombine2 can hack on these. 444 if (Fast) { 445 setConvertAction(MVT::f80, MVT::f32, Expand); 446 setConvertAction(MVT::f64, MVT::f32, Expand); 447 setConvertAction(MVT::f80, MVT::f64, Expand); 448 } 449 450 if (!UnsafeFPMath) { 451 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 452 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 453 } 454 addLegalFPImmediate(APFloat(+0.0)); // FLD0 455 addLegalFPImmediate(APFloat(+1.0)); // FLD1 456 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 457 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 458 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 459 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 460 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 461 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 462 } 463 464 // Long double always uses X87. 465 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 466 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 467 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 468 { 469 APFloat TmpFlt(+0.0); 470 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 471 addLegalFPImmediate(TmpFlt); // FLD0 472 TmpFlt.changeSign(); 473 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 474 APFloat TmpFlt2(+1.0); 475 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 476 addLegalFPImmediate(TmpFlt2); // FLD1 477 TmpFlt2.changeSign(); 478 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 479 } 480 481 if (!UnsafeFPMath) { 482 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 483 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 484 } 485 486 // Always use a library call for pow. 487 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 488 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 489 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 490 491 // First set operation action for all vector types to expand. Then we 492 // will selectively turn on ones that can be effectively codegen'd. 493 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 494 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 495 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 496 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 497 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 498 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 499 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 500 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 501 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 502 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 503 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 504 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 505 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 506 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 507 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 508 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 509 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 510 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 511 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 512 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 513 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 514 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 515 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 516 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 517 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 532 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 533 } 534 535 if (Subtarget->hasMMX()) { 536 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 537 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 538 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 539 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 540 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 541 542 // FIXME: add MMX packed arithmetics 543 544 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 545 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 546 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 547 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 548 549 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 550 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 551 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 552 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 553 554 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 555 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 556 557 setOperationAction(ISD::AND, MVT::v8i8, Promote); 558 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 559 setOperationAction(ISD::AND, MVT::v4i16, Promote); 560 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 561 setOperationAction(ISD::AND, MVT::v2i32, Promote); 562 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 563 setOperationAction(ISD::AND, MVT::v1i64, Legal); 564 565 setOperationAction(ISD::OR, MVT::v8i8, Promote); 566 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 567 setOperationAction(ISD::OR, MVT::v4i16, Promote); 568 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 569 setOperationAction(ISD::OR, MVT::v2i32, Promote); 570 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 571 setOperationAction(ISD::OR, MVT::v1i64, Legal); 572 573 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 574 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 575 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 576 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 577 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 578 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 579 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 580 581 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 582 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 583 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 584 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 585 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 586 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 587 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 588 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 589 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 590 591 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 592 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 593 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 594 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 595 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 596 597 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 598 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 599 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 600 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 601 602 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 603 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 604 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 605 } 606 607 if (Subtarget->hasSSE1()) { 608 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 609 610 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 611 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 612 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 613 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 614 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 615 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 616 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 617 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 618 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 619 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 620 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 621 setOperationAction(ISD::VSETCC, MVT::v4f32, Legal); 622 } 623 624 if (Subtarget->hasSSE2()) { 625 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 626 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 627 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 628 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 629 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 630 631 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 632 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 633 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 634 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 635 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 636 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 637 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 638 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 639 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 640 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 641 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 642 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 643 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 644 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 645 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 646 647 setOperationAction(ISD::VSETCC, MVT::v2f64, Legal); 648 setOperationAction(ISD::VSETCC, MVT::v16i8, Legal); 649 setOperationAction(ISD::VSETCC, MVT::v8i16, Legal); 650 setOperationAction(ISD::VSETCC, MVT::v4i32, Legal); 651 setOperationAction(ISD::VSETCC, MVT::v2i64, Legal); 652 653 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 654 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 655 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 656 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 657 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 658 659 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 660 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 661 MVT VT = (MVT::SimpleValueType)i; 662 // Do not attempt to custom lower non-power-of-2 vectors 663 if (!isPowerOf2_32(VT.getVectorNumElements())) 664 continue; 665 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 666 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 667 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 668 } 669 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 670 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 671 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 672 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 673 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 674 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 675 if (Subtarget->is64Bit()) { 676 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 677 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 678 } 679 680 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 681 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 682 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 683 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 684 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 685 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 686 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 687 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 688 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 689 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 690 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 691 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 692 } 693 694 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 695 696 // Custom lower v2i64 and v2f64 selects. 697 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 698 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 699 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 700 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 701 702 } 703 704 if (Subtarget->hasSSE41()) { 705 // FIXME: Do we need to handle scalar-to-vector here? 706 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 707 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 708 709 // i8 and i16 vectors are custom , because the source register and source 710 // source memory operand types are not the same width. f32 vectors are 711 // custom since the immediate controlling the insert encodes additional 712 // information. 713 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 714 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 715 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 716 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 717 718 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 719 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 720 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 721 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 722 723 if (Subtarget->is64Bit()) { 724 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 725 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 726 } 727 } 728 729 // We want to custom lower some of our intrinsics. 730 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 731 732 // We have target-specific dag combine patterns for the following nodes: 733 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 734 setTargetDAGCombine(ISD::BUILD_VECTOR); 735 setTargetDAGCombine(ISD::SELECT); 736 setTargetDAGCombine(ISD::STORE); 737 738 computeRegisterProperties(); 739 740 // FIXME: These should be based on subtarget info. Plus, the values should 741 // be smaller when we are in optimizing for size mode. 742 maxStoresPerMemset = 16; // For %llvm.memset -> sequence of stores 743 maxStoresPerMemcpy = 16; // For %llvm.memcpy -> sequence of stores 744 maxStoresPerMemmove = 3; // For %llvm.memmove -> sequence of stores 745 allowUnalignedMemoryAccesses = true; // x86 supports it! 746 setPrefLoopAlignment(16); 747} 748 749 750MVT X86TargetLowering::getSetCCResultType(const SDOperand &) const { 751 return MVT::i8; 752} 753 754 755/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 756/// the desired ByVal argument alignment. 757static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 758 if (MaxAlign == 16) 759 return; 760 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 761 if (VTy->getBitWidth() == 128) 762 MaxAlign = 16; 763 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 764 unsigned EltAlign = 0; 765 getMaxByValAlign(ATy->getElementType(), EltAlign); 766 if (EltAlign > MaxAlign) 767 MaxAlign = EltAlign; 768 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 769 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 770 unsigned EltAlign = 0; 771 getMaxByValAlign(STy->getElementType(i), EltAlign); 772 if (EltAlign > MaxAlign) 773 MaxAlign = EltAlign; 774 if (MaxAlign == 16) 775 break; 776 } 777 } 778 return; 779} 780 781/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 782/// function arguments in the caller parameter area. For X86, aggregates 783/// that contain SSE vectors are placed at 16-byte boundaries while the rest 784/// are at 4-byte boundaries. 785unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 786 if (Subtarget->is64Bit()) 787 return getTargetData()->getABITypeAlignment(Ty); 788 unsigned Align = 4; 789 if (Subtarget->hasSSE1()) 790 getMaxByValAlign(Ty, Align); 791 return Align; 792} 793 794/// getOptimalMemOpType - Returns the target specific optimal type for load 795/// and store operations as a result of memset, memcpy, and memmove 796/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 797/// determining it. 798MVT 799X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 800 bool isSrcConst, bool isSrcStr) const { 801 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 802 return MVT::v4i32; 803 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 804 return MVT::v4f32; 805 if (Subtarget->is64Bit() && Size >= 8) 806 return MVT::i64; 807 return MVT::i32; 808} 809 810 811/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 812/// jumptable. 813SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table, 814 SelectionDAG &DAG) const { 815 if (usesGlobalOffsetTable()) 816 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 817 if (!Subtarget->isPICStyleRIPRel()) 818 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 819 return Table; 820} 821 822//===----------------------------------------------------------------------===// 823// Return Value Calling Convention Implementation 824//===----------------------------------------------------------------------===// 825 826#include "X86GenCallingConv.inc" 827 828/// LowerRET - Lower an ISD::RET node. 829SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) { 830 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 831 832 SmallVector<CCValAssign, 16> RVLocs; 833 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 834 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 835 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 836 CCInfo.AnalyzeReturn(Op.Val, RetCC_X86); 837 838 // If this is the first return lowered for this function, add the regs to the 839 // liveout set for the function. 840 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 841 for (unsigned i = 0; i != RVLocs.size(); ++i) 842 if (RVLocs[i].isRegLoc()) 843 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 844 } 845 SDOperand Chain = Op.getOperand(0); 846 847 // Handle tail call return. 848 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 849 if (Chain.getOpcode() == X86ISD::TAILCALL) { 850 SDOperand TailCall = Chain; 851 SDOperand TargetAddress = TailCall.getOperand(1); 852 SDOperand StackAdjustment = TailCall.getOperand(2); 853 assert(((TargetAddress.getOpcode() == ISD::Register && 854 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 855 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 856 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 857 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 858 "Expecting an global address, external symbol, or register"); 859 assert(StackAdjustment.getOpcode() == ISD::Constant && 860 "Expecting a const value"); 861 862 SmallVector<SDOperand,8> Operands; 863 Operands.push_back(Chain.getOperand(0)); 864 Operands.push_back(TargetAddress); 865 Operands.push_back(StackAdjustment); 866 // Copy registers used by the call. Last operand is a flag so it is not 867 // copied. 868 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 869 Operands.push_back(Chain.getOperand(i)); 870 } 871 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 872 Operands.size()); 873 } 874 875 // Regular return. 876 SDOperand Flag; 877 878 SmallVector<SDOperand, 6> RetOps; 879 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 880 // Operand #1 = Bytes To Pop 881 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 882 883 // Copy the result values into the output registers. 884 for (unsigned i = 0; i != RVLocs.size(); ++i) { 885 CCValAssign &VA = RVLocs[i]; 886 assert(VA.isRegLoc() && "Can only return in registers!"); 887 SDOperand ValToCopy = Op.getOperand(i*2+1); 888 889 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 890 // the RET instruction and handled by the FP Stackifier. 891 if (RVLocs[i].getLocReg() == X86::ST0 || 892 RVLocs[i].getLocReg() == X86::ST1) { 893 // If this is a copy from an xmm register to ST(0), use an FPExtend to 894 // change the value to the FP stack register class. 895 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 896 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 897 RetOps.push_back(ValToCopy); 898 // Don't emit a copytoreg. 899 continue; 900 } 901 902 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 903 Flag = Chain.getValue(1); 904 } 905 906 // The x86-64 ABI for returning structs by value requires that we copy 907 // the sret argument into %rax for the return. We saved the argument into 908 // a virtual register in the entry block, so now we copy the value out 909 // and into %rax. 910 if (Subtarget->is64Bit() && 911 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 912 MachineFunction &MF = DAG.getMachineFunction(); 913 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 914 unsigned Reg = FuncInfo->getSRetReturnReg(); 915 if (!Reg) { 916 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 917 FuncInfo->setSRetReturnReg(Reg); 918 } 919 SDOperand Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 920 921 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 922 Flag = Chain.getValue(1); 923 } 924 925 RetOps[0] = Chain; // Update chain. 926 927 // Add the flag if we have it. 928 if (Flag.Val) 929 RetOps.push_back(Flag); 930 931 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 932} 933 934 935/// LowerCallResult - Lower the result values of an ISD::CALL into the 936/// appropriate copies out of appropriate physical registers. This assumes that 937/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 938/// being lowered. The returns a SDNode with the same number of values as the 939/// ISD::CALL. 940SDNode *X86TargetLowering:: 941LowerCallResult(SDOperand Chain, SDOperand InFlag, SDNode *TheCall, 942 unsigned CallingConv, SelectionDAG &DAG) { 943 944 // Assign locations to each value returned by this call. 945 SmallVector<CCValAssign, 16> RVLocs; 946 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 947 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 948 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 949 950 SmallVector<SDOperand, 8> ResultVals; 951 952 // Copy all of the result registers out of their specified physreg. 953 for (unsigned i = 0; i != RVLocs.size(); ++i) { 954 MVT CopyVT = RVLocs[i].getValVT(); 955 956 // If this is a call to a function that returns an fp value on the floating 957 // point stack, but where we prefer to use the value in xmm registers, copy 958 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 959 if (RVLocs[i].getLocReg() == X86::ST0 && 960 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 961 CopyVT = MVT::f80; 962 } 963 964 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 965 CopyVT, InFlag).getValue(1); 966 SDOperand Val = Chain.getValue(0); 967 InFlag = Chain.getValue(2); 968 969 if (CopyVT != RVLocs[i].getValVT()) { 970 // Round the F80 the right size, which also moves to the appropriate xmm 971 // register. 972 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 973 // This truncation won't change the value. 974 DAG.getIntPtrConstant(1)); 975 } 976 977 ResultVals.push_back(Val); 978 } 979 980 // Merge everything together with a MERGE_VALUES node. 981 ResultVals.push_back(Chain); 982 return DAG.getNode(ISD::MERGE_VALUES, TheCall->getVTList(), 983 &ResultVals[0], ResultVals.size()).Val; 984} 985 986 987//===----------------------------------------------------------------------===// 988// C & StdCall & Fast Calling Convention implementation 989//===----------------------------------------------------------------------===// 990// StdCall calling convention seems to be standard for many Windows' API 991// routines and around. It differs from C calling convention just a little: 992// callee should clean up the stack, not caller. Symbols should be also 993// decorated in some fancy way :) It doesn't support any vector arguments. 994// For info on fast calling convention see Fast Calling Convention (tail call) 995// implementation LowerX86_32FastCCCallTo. 996 997/// AddLiveIn - This helper function adds the specified physical register to the 998/// MachineFunction as a live in value. It also creates a corresponding virtual 999/// register for it. 1000static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1001 const TargetRegisterClass *RC) { 1002 assert(RC->contains(PReg) && "Not the correct regclass!"); 1003 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1004 MF.getRegInfo().addLiveIn(PReg, VReg); 1005 return VReg; 1006} 1007 1008/// CallIsStructReturn - Determines whether a CALL node uses struct return 1009/// semantics. 1010static bool CallIsStructReturn(SDOperand Op) { 1011 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1012 if (!NumOps) 1013 return false; 1014 1015 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1016} 1017 1018/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1019/// return semantics. 1020static bool ArgsAreStructReturn(SDOperand Op) { 1021 unsigned NumArgs = Op.Val->getNumValues() - 1; 1022 if (!NumArgs) 1023 return false; 1024 1025 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1026} 1027 1028/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1029/// the callee to pop its own arguments. Callee pop is necessary to support tail 1030/// calls. 1031bool X86TargetLowering::IsCalleePop(SDOperand Op) { 1032 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1033 if (IsVarArg) 1034 return false; 1035 1036 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1037 default: 1038 return false; 1039 case CallingConv::X86_StdCall: 1040 return !Subtarget->is64Bit(); 1041 case CallingConv::X86_FastCall: 1042 return !Subtarget->is64Bit(); 1043 case CallingConv::Fast: 1044 return PerformTailCallOpt; 1045 } 1046} 1047 1048/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1049/// FORMAL_ARGUMENTS node. 1050CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDOperand Op) const { 1051 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1052 1053 if (Subtarget->is64Bit()) { 1054 if (Subtarget->isTargetWin64()) 1055 return CC_X86_Win64_C; 1056 else { 1057 if (CC == CallingConv::Fast && PerformTailCallOpt) 1058 return CC_X86_64_TailCall; 1059 else 1060 return CC_X86_64_C; 1061 } 1062 } 1063 1064 if (CC == CallingConv::X86_FastCall) 1065 return CC_X86_32_FastCall; 1066 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1067 return CC_X86_32_TailCall; 1068 else 1069 return CC_X86_32_C; 1070} 1071 1072/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1073/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1074NameDecorationStyle 1075X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDOperand Op) { 1076 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1077 if (CC == CallingConv::X86_FastCall) 1078 return FastCall; 1079 else if (CC == CallingConv::X86_StdCall) 1080 return StdCall; 1081 return None; 1082} 1083 1084 1085/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1086/// in a register before calling. 1087bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1088 return !IsTailCall && !Is64Bit && 1089 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1090 Subtarget->isPICStyleGOT(); 1091} 1092 1093/// CallRequiresFnAddressInReg - Check whether the call requires the function 1094/// address to be loaded in a register. 1095bool 1096X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1097 return !Is64Bit && IsTailCall && 1098 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1099 Subtarget->isPICStyleGOT(); 1100} 1101 1102/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1103/// by "Src" to address "Dst" with size and alignment information specified by 1104/// the specific parameter attribute. The copy will be passed as a byval 1105/// function parameter. 1106static SDOperand 1107CreateCopyOfByValArgument(SDOperand Src, SDOperand Dst, SDOperand Chain, 1108 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1109 SDOperand SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1110 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1111 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1112} 1113 1114SDOperand X86TargetLowering::LowerMemArgument(SDOperand Op, SelectionDAG &DAG, 1115 const CCValAssign &VA, 1116 MachineFrameInfo *MFI, 1117 unsigned CC, 1118 SDOperand Root, unsigned i) { 1119 // Create the nodes corresponding to a load from this parameter slot. 1120 ISD::ArgFlagsTy Flags = 1121 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1122 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1123 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1124 1125 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1126 // changed with more analysis. 1127 // In case of tail call optimization mark all arguments mutable. Since they 1128 // could be overwritten by lowering of arguments in case of a tail call. 1129 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1130 VA.getLocMemOffset(), isImmutable); 1131 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); 1132 if (Flags.isByVal()) 1133 return FIN; 1134 return DAG.getLoad(VA.getValVT(), Root, FIN, 1135 PseudoSourceValue::getFixedStack(), FI); 1136} 1137 1138SDOperand 1139X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) { 1140 MachineFunction &MF = DAG.getMachineFunction(); 1141 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1142 1143 const Function* Fn = MF.getFunction(); 1144 if (Fn->hasExternalLinkage() && 1145 Subtarget->isTargetCygMing() && 1146 Fn->getName() == "main") 1147 FuncInfo->setForceFramePointer(true); 1148 1149 // Decorate the function name. 1150 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1151 1152 MachineFrameInfo *MFI = MF.getFrameInfo(); 1153 SDOperand Root = Op.getOperand(0); 1154 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1155 unsigned CC = MF.getFunction()->getCallingConv(); 1156 bool Is64Bit = Subtarget->is64Bit(); 1157 bool IsWin64 = Subtarget->isTargetWin64(); 1158 1159 assert(!(isVarArg && CC == CallingConv::Fast) && 1160 "Var args not supported with calling convention fastcc"); 1161 1162 // Assign locations to all of the incoming arguments. 1163 SmallVector<CCValAssign, 16> ArgLocs; 1164 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1165 CCInfo.AnalyzeFormalArguments(Op.Val, CCAssignFnForNode(Op)); 1166 1167 SmallVector<SDOperand, 8> ArgValues; 1168 unsigned LastVal = ~0U; 1169 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1170 CCValAssign &VA = ArgLocs[i]; 1171 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1172 // places. 1173 assert(VA.getValNo() != LastVal && 1174 "Don't support value assigned to multiple locs yet"); 1175 LastVal = VA.getValNo(); 1176 1177 if (VA.isRegLoc()) { 1178 MVT RegVT = VA.getLocVT(); 1179 TargetRegisterClass *RC; 1180 if (RegVT == MVT::i32) 1181 RC = X86::GR32RegisterClass; 1182 else if (Is64Bit && RegVT == MVT::i64) 1183 RC = X86::GR64RegisterClass; 1184 else if (RegVT == MVT::f32) 1185 RC = X86::FR32RegisterClass; 1186 else if (RegVT == MVT::f64) 1187 RC = X86::FR64RegisterClass; 1188 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1189 RC = X86::VR128RegisterClass; 1190 else if (RegVT.isVector()) { 1191 assert(RegVT.getSizeInBits() == 64); 1192 if (!Is64Bit) 1193 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1194 else { 1195 // Darwin calling convention passes MMX values in either GPRs or 1196 // XMMs in x86-64. Other targets pass them in memory. 1197 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1198 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1199 RegVT = MVT::v2i64; 1200 } else { 1201 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1202 RegVT = MVT::i64; 1203 } 1204 } 1205 } else { 1206 assert(0 && "Unknown argument type!"); 1207 } 1208 1209 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1210 SDOperand ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1211 1212 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1213 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1214 // right size. 1215 if (VA.getLocInfo() == CCValAssign::SExt) 1216 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1217 DAG.getValueType(VA.getValVT())); 1218 else if (VA.getLocInfo() == CCValAssign::ZExt) 1219 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1220 DAG.getValueType(VA.getValVT())); 1221 1222 if (VA.getLocInfo() != CCValAssign::Full) 1223 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1224 1225 // Handle MMX values passed in GPRs. 1226 if (Is64Bit && RegVT != VA.getLocVT()) { 1227 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1228 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1229 else if (RC == X86::VR128RegisterClass) { 1230 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1231 DAG.getConstant(0, MVT::i64)); 1232 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1233 } 1234 } 1235 1236 ArgValues.push_back(ArgValue); 1237 } else { 1238 assert(VA.isMemLoc()); 1239 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1240 } 1241 } 1242 1243 // The x86-64 ABI for returning structs by value requires that we copy 1244 // the sret argument into %rax for the return. Save the argument into 1245 // a virtual register so that we can access it from the return points. 1246 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1247 MachineFunction &MF = DAG.getMachineFunction(); 1248 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1249 unsigned Reg = FuncInfo->getSRetReturnReg(); 1250 if (!Reg) { 1251 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1252 FuncInfo->setSRetReturnReg(Reg); 1253 } 1254 SDOperand Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1255 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1256 } 1257 1258 unsigned StackSize = CCInfo.getNextStackOffset(); 1259 // align stack specially for tail calls 1260 if (CC == CallingConv::Fast) 1261 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1262 1263 // If the function takes variable number of arguments, make a frame index for 1264 // the start of the first vararg value... for expansion of llvm.va_start. 1265 if (isVarArg) { 1266 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1267 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1268 } 1269 if (Is64Bit) { 1270 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1271 1272 // FIXME: We should really autogenerate these arrays 1273 static const unsigned GPR64ArgRegsWin64[] = { 1274 X86::RCX, X86::RDX, X86::R8, X86::R9 1275 }; 1276 static const unsigned XMMArgRegsWin64[] = { 1277 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1278 }; 1279 static const unsigned GPR64ArgRegs64Bit[] = { 1280 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1281 }; 1282 static const unsigned XMMArgRegs64Bit[] = { 1283 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1284 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1285 }; 1286 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1287 1288 if (IsWin64) { 1289 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1290 GPR64ArgRegs = GPR64ArgRegsWin64; 1291 XMMArgRegs = XMMArgRegsWin64; 1292 } else { 1293 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1294 GPR64ArgRegs = GPR64ArgRegs64Bit; 1295 XMMArgRegs = XMMArgRegs64Bit; 1296 } 1297 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1298 TotalNumIntRegs); 1299 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1300 TotalNumXMMRegs); 1301 1302 // For X86-64, if there are vararg parameters that are passed via 1303 // registers, then we must store them to their spots on the stack so they 1304 // may be loaded by deferencing the result of va_next. 1305 VarArgsGPOffset = NumIntRegs * 8; 1306 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1307 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1308 TotalNumXMMRegs * 16, 16); 1309 1310 // Store the integer parameter registers. 1311 SmallVector<SDOperand, 8> MemOps; 1312 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1313 SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1314 DAG.getIntPtrConstant(VarArgsGPOffset)); 1315 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1316 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1317 X86::GR64RegisterClass); 1318 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1319 SDOperand Store = 1320 DAG.getStore(Val.getValue(1), Val, FIN, 1321 PseudoSourceValue::getFixedStack(), 1322 RegSaveFrameIndex); 1323 MemOps.push_back(Store); 1324 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1325 DAG.getIntPtrConstant(8)); 1326 } 1327 1328 // Now store the XMM (fp + vector) parameter registers. 1329 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1330 DAG.getIntPtrConstant(VarArgsFPOffset)); 1331 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1332 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1333 X86::VR128RegisterClass); 1334 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1335 SDOperand Store = 1336 DAG.getStore(Val.getValue(1), Val, FIN, 1337 PseudoSourceValue::getFixedStack(), 1338 RegSaveFrameIndex); 1339 MemOps.push_back(Store); 1340 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1341 DAG.getIntPtrConstant(16)); 1342 } 1343 if (!MemOps.empty()) 1344 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1345 &MemOps[0], MemOps.size()); 1346 } 1347 } 1348 1349 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1350 // arguments and the arguments after the retaddr has been pushed are 1351 // aligned. 1352 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1353 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1354 (StackSize & 7) == 0) 1355 StackSize += 4; 1356 1357 ArgValues.push_back(Root); 1358 1359 // Some CCs need callee pop. 1360 if (IsCalleePop(Op)) { 1361 BytesToPopOnReturn = StackSize; // Callee pops everything. 1362 BytesCallerReserves = 0; 1363 } else { 1364 BytesToPopOnReturn = 0; // Callee pops nothing. 1365 // If this is an sret function, the return should pop the hidden pointer. 1366 if (!Is64Bit && ArgsAreStructReturn(Op)) 1367 BytesToPopOnReturn = 4; 1368 BytesCallerReserves = StackSize; 1369 } 1370 1371 if (!Is64Bit) { 1372 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1373 if (CC == CallingConv::X86_FastCall) 1374 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1375 } 1376 1377 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1378 1379 // Return the new list of results. 1380 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(), 1381 &ArgValues[0], ArgValues.size()).getValue(Op.ResNo); 1382} 1383 1384SDOperand 1385X86TargetLowering::LowerMemOpCallTo(SDOperand Op, SelectionDAG &DAG, 1386 const SDOperand &StackPtr, 1387 const CCValAssign &VA, 1388 SDOperand Chain, 1389 SDOperand Arg) { 1390 unsigned LocMemOffset = VA.getLocMemOffset(); 1391 SDOperand PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1392 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1393 ISD::ArgFlagsTy Flags = 1394 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1395 if (Flags.isByVal()) { 1396 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1397 } 1398 return DAG.getStore(Chain, Arg, PtrOff, 1399 PseudoSourceValue::getStack(), LocMemOffset); 1400} 1401 1402/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1403/// optimization is performed and it is required. 1404SDOperand 1405X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1406 SDOperand &OutRetAddr, 1407 SDOperand Chain, 1408 bool IsTailCall, 1409 bool Is64Bit, 1410 int FPDiff) { 1411 if (!IsTailCall || FPDiff==0) return Chain; 1412 1413 // Adjust the Return address stack slot. 1414 MVT VT = getPointerTy(); 1415 OutRetAddr = getReturnAddressFrameIndex(DAG); 1416 // Load the "old" Return address. 1417 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1418 return SDOperand(OutRetAddr.Val, 1); 1419} 1420 1421/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1422/// optimization is performed and it is required (FPDiff!=0). 1423static SDOperand 1424EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1425 SDOperand Chain, SDOperand RetAddrFrIdx, 1426 bool Is64Bit, int FPDiff) { 1427 // Store the return address to the appropriate stack slot. 1428 if (!FPDiff) return Chain; 1429 // Calculate the new stack slot for the return address. 1430 int SlotSize = Is64Bit ? 8 : 4; 1431 int NewReturnAddrFI = 1432 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1433 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1434 SDOperand NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1435 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1436 PseudoSourceValue::getFixedStack(), NewReturnAddrFI); 1437 return Chain; 1438} 1439 1440SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) { 1441 MachineFunction &MF = DAG.getMachineFunction(); 1442 SDOperand Chain = Op.getOperand(0); 1443 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1444 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1445 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1446 && CC == CallingConv::Fast && PerformTailCallOpt; 1447 SDOperand Callee = Op.getOperand(4); 1448 bool Is64Bit = Subtarget->is64Bit(); 1449 bool IsStructRet = CallIsStructReturn(Op); 1450 1451 assert(!(isVarArg && CC == CallingConv::Fast) && 1452 "Var args not supported with calling convention fastcc"); 1453 1454 // Analyze operands of the call, assigning locations to each operand. 1455 SmallVector<CCValAssign, 16> ArgLocs; 1456 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1457 CCInfo.AnalyzeCallOperands(Op.Val, CCAssignFnForNode(Op)); 1458 1459 // Get a count of how many bytes are to be pushed on the stack. 1460 unsigned NumBytes = CCInfo.getNextStackOffset(); 1461 if (CC == CallingConv::Fast) 1462 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1463 1464 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1465 // arguments and the arguments after the retaddr has been pushed are aligned. 1466 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1467 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1468 (NumBytes & 7) == 0) 1469 NumBytes += 4; 1470 1471 int FPDiff = 0; 1472 if (IsTailCall) { 1473 // Lower arguments at fp - stackoffset + fpdiff. 1474 unsigned NumBytesCallerPushed = 1475 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1476 FPDiff = NumBytesCallerPushed - NumBytes; 1477 1478 // Set the delta of movement of the returnaddr stackslot. 1479 // But only set if delta is greater than previous delta. 1480 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1481 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1482 } 1483 1484 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1485 1486 SDOperand RetAddrFrIdx; 1487 // Load return adress for tail calls. 1488 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1489 FPDiff); 1490 1491 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass; 1492 SmallVector<SDOperand, 8> MemOpChains; 1493 SDOperand StackPtr; 1494 1495 // Walk the register/memloc assignments, inserting copies/loads. In the case 1496 // of tail call optimization arguments are handle later. 1497 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1498 CCValAssign &VA = ArgLocs[i]; 1499 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1500 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1501 getArgFlags().isByVal(); 1502 1503 // Promote the value if needed. 1504 switch (VA.getLocInfo()) { 1505 default: assert(0 && "Unknown loc info!"); 1506 case CCValAssign::Full: break; 1507 case CCValAssign::SExt: 1508 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1509 break; 1510 case CCValAssign::ZExt: 1511 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1512 break; 1513 case CCValAssign::AExt: 1514 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1515 break; 1516 } 1517 1518 if (VA.isRegLoc()) { 1519 if (Is64Bit) { 1520 MVT RegVT = VA.getLocVT(); 1521 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1522 switch (VA.getLocReg()) { 1523 default: 1524 break; 1525 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1526 case X86::R8: { 1527 // Special case: passing MMX values in GPR registers. 1528 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1529 break; 1530 } 1531 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1532 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1533 // Special case: passing MMX values in XMM registers. 1534 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1535 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1536 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1537 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1538 getMOVLMask(2, DAG)); 1539 break; 1540 } 1541 } 1542 } 1543 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1544 } else { 1545 if (!IsTailCall || (IsTailCall && isByVal)) { 1546 assert(VA.isMemLoc()); 1547 if (StackPtr.Val == 0) 1548 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1549 1550 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1551 Arg)); 1552 } 1553 } 1554 } 1555 1556 if (!MemOpChains.empty()) 1557 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1558 &MemOpChains[0], MemOpChains.size()); 1559 1560 // Build a sequence of copy-to-reg nodes chained together with token chain 1561 // and flag operands which copy the outgoing args into registers. 1562 SDOperand InFlag; 1563 // Tail call byval lowering might overwrite argument registers so in case of 1564 // tail call optimization the copies to registers are lowered later. 1565 if (!IsTailCall) 1566 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1567 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1568 InFlag); 1569 InFlag = Chain.getValue(1); 1570 } 1571 1572 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1573 // GOT pointer. 1574 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1575 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1576 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1577 InFlag); 1578 InFlag = Chain.getValue(1); 1579 } 1580 // If we are tail calling and generating PIC/GOT style code load the address 1581 // of the callee into ecx. The value in ecx is used as target of the tail 1582 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1583 // calls on PIC/GOT architectures. Normally we would just put the address of 1584 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1585 // restored (since ebx is callee saved) before jumping to the target@PLT. 1586 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1587 // Note: The actual moving to ecx is done further down. 1588 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1589 if (G && !G->getGlobal()->hasHiddenVisibility() && 1590 !G->getGlobal()->hasProtectedVisibility()) 1591 Callee = LowerGlobalAddress(Callee, DAG); 1592 else if (isa<ExternalSymbolSDNode>(Callee)) 1593 Callee = LowerExternalSymbol(Callee,DAG); 1594 } 1595 1596 if (Is64Bit && isVarArg) { 1597 // From AMD64 ABI document: 1598 // For calls that may call functions that use varargs or stdargs 1599 // (prototype-less calls or calls to functions containing ellipsis (...) in 1600 // the declaration) %al is used as hidden argument to specify the number 1601 // of SSE registers used. The contents of %al do not need to match exactly 1602 // the number of registers, but must be an ubound on the number of SSE 1603 // registers used and is in the range 0 - 8 inclusive. 1604 1605 // FIXME: Verify this on Win64 1606 // Count the number of XMM registers allocated. 1607 static const unsigned XMMArgRegs[] = { 1608 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1609 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1610 }; 1611 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1612 1613 Chain = DAG.getCopyToReg(Chain, X86::AL, 1614 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1615 InFlag = Chain.getValue(1); 1616 } 1617 1618 1619 // For tail calls lower the arguments to the 'real' stack slot. 1620 if (IsTailCall) { 1621 SmallVector<SDOperand, 8> MemOpChains2; 1622 SDOperand FIN; 1623 int FI = 0; 1624 // Do not flag preceeding copytoreg stuff together with the following stuff. 1625 InFlag = SDOperand(); 1626 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1627 CCValAssign &VA = ArgLocs[i]; 1628 if (!VA.isRegLoc()) { 1629 assert(VA.isMemLoc()); 1630 SDOperand Arg = Op.getOperand(5+2*VA.getValNo()); 1631 SDOperand FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1632 ISD::ArgFlagsTy Flags = 1633 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1634 // Create frame index. 1635 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1636 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1637 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1638 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1639 1640 if (Flags.isByVal()) { 1641 // Copy relative to framepointer. 1642 SDOperand Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1643 if (StackPtr.Val == 0) 1644 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1645 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1646 1647 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1648 Flags, DAG)); 1649 } else { 1650 // Store relative to framepointer. 1651 MemOpChains2.push_back( 1652 DAG.getStore(Chain, Arg, FIN, 1653 PseudoSourceValue::getFixedStack(), FI)); 1654 } 1655 } 1656 } 1657 1658 if (!MemOpChains2.empty()) 1659 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1660 &MemOpChains2[0], MemOpChains2.size()); 1661 1662 // Copy arguments to their registers. 1663 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1664 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1665 InFlag); 1666 InFlag = Chain.getValue(1); 1667 } 1668 InFlag =SDOperand(); 1669 1670 // Store the return address to the appropriate stack slot. 1671 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1672 FPDiff); 1673 } 1674 1675 // If the callee is a GlobalAddress node (quite common, every direct call is) 1676 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1677 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1678 // We should use extra load for direct calls to dllimported functions in 1679 // non-JIT mode. 1680 if ((IsTailCall || !Is64Bit || 1681 getTargetMachine().getCodeModel() != CodeModel::Large) 1682 && !Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1683 getTargetMachine(), true)) 1684 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1685 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1686 if (IsTailCall || !Is64Bit || 1687 getTargetMachine().getCodeModel() != CodeModel::Large) 1688 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1689 } else if (IsTailCall) { 1690 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1691 1692 Chain = DAG.getCopyToReg(Chain, 1693 DAG.getRegister(Opc, getPointerTy()), 1694 Callee,InFlag); 1695 Callee = DAG.getRegister(Opc, getPointerTy()); 1696 // Add register as live out. 1697 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1698 } 1699 1700 // Returns a chain & a flag for retval copy to use. 1701 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1702 SmallVector<SDOperand, 8> Ops; 1703 1704 if (IsTailCall) { 1705 Ops.push_back(Chain); 1706 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1707 Ops.push_back(DAG.getIntPtrConstant(0)); 1708 if (InFlag.Val) 1709 Ops.push_back(InFlag); 1710 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1711 InFlag = Chain.getValue(1); 1712 1713 // Returns a chain & a flag for retval copy to use. 1714 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1715 Ops.clear(); 1716 } 1717 1718 Ops.push_back(Chain); 1719 Ops.push_back(Callee); 1720 1721 if (IsTailCall) 1722 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1723 1724 // Add argument registers to the end of the list so that they are known live 1725 // into the call. 1726 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1727 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1728 RegsToPass[i].second.getValueType())); 1729 1730 // Add an implicit use GOT pointer in EBX. 1731 if (!IsTailCall && !Is64Bit && 1732 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1733 Subtarget->isPICStyleGOT()) 1734 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1735 1736 // Add an implicit use of AL for x86 vararg functions. 1737 if (Is64Bit && isVarArg) 1738 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1739 1740 if (InFlag.Val) 1741 Ops.push_back(InFlag); 1742 1743 if (IsTailCall) { 1744 assert(InFlag.Val && 1745 "Flag must be set. Depend on flag being set in LowerRET"); 1746 Chain = DAG.getNode(X86ISD::TAILCALL, 1747 Op.Val->getVTList(), &Ops[0], Ops.size()); 1748 1749 return SDOperand(Chain.Val, Op.ResNo); 1750 } 1751 1752 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1753 InFlag = Chain.getValue(1); 1754 1755 // Create the CALLSEQ_END node. 1756 unsigned NumBytesForCalleeToPush; 1757 if (IsCalleePop(Op)) 1758 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1759 else if (!Is64Bit && IsStructRet) 1760 // If this is is a call to a struct-return function, the callee 1761 // pops the hidden struct pointer, so we have to push it back. 1762 // This is common for Darwin/X86, Linux & Mingw32 targets. 1763 NumBytesForCalleeToPush = 4; 1764 else 1765 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1766 1767 // Returns a flag for retval copy to use. 1768 Chain = DAG.getCALLSEQ_END(Chain, 1769 DAG.getIntPtrConstant(NumBytes), 1770 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1771 InFlag); 1772 InFlag = Chain.getValue(1); 1773 1774 // Handle result values, copying them out of physregs into vregs that we 1775 // return. 1776 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo); 1777} 1778 1779 1780//===----------------------------------------------------------------------===// 1781// Fast Calling Convention (tail call) implementation 1782//===----------------------------------------------------------------------===// 1783 1784// Like std call, callee cleans arguments, convention except that ECX is 1785// reserved for storing the tail called function address. Only 2 registers are 1786// free for argument passing (inreg). Tail call optimization is performed 1787// provided: 1788// * tailcallopt is enabled 1789// * caller/callee are fastcc 1790// On X86_64 architecture with GOT-style position independent code only local 1791// (within module) calls are supported at the moment. 1792// To keep the stack aligned according to platform abi the function 1793// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1794// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1795// If a tail called function callee has more arguments than the caller the 1796// caller needs to make sure that there is room to move the RETADDR to. This is 1797// achieved by reserving an area the size of the argument delta right after the 1798// original REtADDR, but before the saved framepointer or the spilled registers 1799// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1800// stack layout: 1801// arg1 1802// arg2 1803// RETADDR 1804// [ new RETADDR 1805// move area ] 1806// (possible EBP) 1807// ESI 1808// EDI 1809// local1 .. 1810 1811/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1812/// for a 16 byte align requirement. 1813unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1814 SelectionDAG& DAG) { 1815 if (PerformTailCallOpt) { 1816 MachineFunction &MF = DAG.getMachineFunction(); 1817 const TargetMachine &TM = MF.getTarget(); 1818 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1819 unsigned StackAlignment = TFI.getStackAlignment(); 1820 uint64_t AlignMask = StackAlignment - 1; 1821 int64_t Offset = StackSize; 1822 unsigned SlotSize = Subtarget->is64Bit() ? 8 : 4; 1823 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1824 // Number smaller than 12 so just add the difference. 1825 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1826 } else { 1827 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1828 Offset = ((~AlignMask) & Offset) + StackAlignment + 1829 (StackAlignment-SlotSize); 1830 } 1831 StackSize = Offset; 1832 } 1833 return StackSize; 1834} 1835 1836/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1837/// following the call is a return. A function is eligible if caller/callee 1838/// calling conventions match, currently only fastcc supports tail calls, and 1839/// the function CALL is immediatly followed by a RET. 1840bool X86TargetLowering::IsEligibleForTailCallOptimization(SDOperand Call, 1841 SDOperand Ret, 1842 SelectionDAG& DAG) const { 1843 if (!PerformTailCallOpt) 1844 return false; 1845 1846 if (CheckTailCallReturnConstraints(Call, Ret)) { 1847 MachineFunction &MF = DAG.getMachineFunction(); 1848 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1849 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1850 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1851 SDOperand Callee = Call.getOperand(4); 1852 // On x86/32Bit PIC/GOT tail calls are supported. 1853 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1854 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1855 return true; 1856 1857 // Can only do local tail calls (in same module, hidden or protected) on 1858 // x86_64 PIC/GOT at the moment. 1859 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1860 return G->getGlobal()->hasHiddenVisibility() 1861 || G->getGlobal()->hasProtectedVisibility(); 1862 } 1863 } 1864 1865 return false; 1866} 1867 1868//===----------------------------------------------------------------------===// 1869// Other Lowering Hooks 1870//===----------------------------------------------------------------------===// 1871 1872 1873SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1874 MachineFunction &MF = DAG.getMachineFunction(); 1875 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1876 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1877 1878 if (ReturnAddrIndex == 0) { 1879 // Set up a frame object for the return address. 1880 if (Subtarget->is64Bit()) 1881 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); 1882 else 1883 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1884 1885 FuncInfo->setRAIndex(ReturnAddrIndex); 1886 } 1887 1888 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1889} 1890 1891 1892 1893/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1894/// specific condition code. It returns a false if it cannot do a direct 1895/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1896/// needed. 1897static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1898 unsigned &X86CC, SDOperand &LHS, SDOperand &RHS, 1899 SelectionDAG &DAG) { 1900 X86CC = X86::COND_INVALID; 1901 if (!isFP) { 1902 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1903 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1904 // X > -1 -> X == 0, jump !sign. 1905 RHS = DAG.getConstant(0, RHS.getValueType()); 1906 X86CC = X86::COND_NS; 1907 return true; 1908 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1909 // X < 0 -> X == 0, jump on sign. 1910 X86CC = X86::COND_S; 1911 return true; 1912 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1913 // X < 1 -> X <= 0 1914 RHS = DAG.getConstant(0, RHS.getValueType()); 1915 X86CC = X86::COND_LE; 1916 return true; 1917 } 1918 } 1919 1920 switch (SetCCOpcode) { 1921 default: break; 1922 case ISD::SETEQ: X86CC = X86::COND_E; break; 1923 case ISD::SETGT: X86CC = X86::COND_G; break; 1924 case ISD::SETGE: X86CC = X86::COND_GE; break; 1925 case ISD::SETLT: X86CC = X86::COND_L; break; 1926 case ISD::SETLE: X86CC = X86::COND_LE; break; 1927 case ISD::SETNE: X86CC = X86::COND_NE; break; 1928 case ISD::SETULT: X86CC = X86::COND_B; break; 1929 case ISD::SETUGT: X86CC = X86::COND_A; break; 1930 case ISD::SETULE: X86CC = X86::COND_BE; break; 1931 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1932 } 1933 } else { 1934 // On a floating point condition, the flags are set as follows: 1935 // ZF PF CF op 1936 // 0 | 0 | 0 | X > Y 1937 // 0 | 0 | 1 | X < Y 1938 // 1 | 0 | 0 | X == Y 1939 // 1 | 1 | 1 | unordered 1940 bool Flip = false; 1941 switch (SetCCOpcode) { 1942 default: break; 1943 case ISD::SETUEQ: 1944 case ISD::SETEQ: X86CC = X86::COND_E; break; 1945 case ISD::SETOLT: Flip = true; // Fallthrough 1946 case ISD::SETOGT: 1947 case ISD::SETGT: X86CC = X86::COND_A; break; 1948 case ISD::SETOLE: Flip = true; // Fallthrough 1949 case ISD::SETOGE: 1950 case ISD::SETGE: X86CC = X86::COND_AE; break; 1951 case ISD::SETUGT: Flip = true; // Fallthrough 1952 case ISD::SETULT: 1953 case ISD::SETLT: X86CC = X86::COND_B; break; 1954 case ISD::SETUGE: Flip = true; // Fallthrough 1955 case ISD::SETULE: 1956 case ISD::SETLE: X86CC = X86::COND_BE; break; 1957 case ISD::SETONE: 1958 case ISD::SETNE: X86CC = X86::COND_NE; break; 1959 case ISD::SETUO: X86CC = X86::COND_P; break; 1960 case ISD::SETO: X86CC = X86::COND_NP; break; 1961 } 1962 if (Flip) 1963 std::swap(LHS, RHS); 1964 } 1965 1966 return X86CC != X86::COND_INVALID; 1967} 1968 1969/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1970/// code. Current x86 isa includes the following FP cmov instructions: 1971/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1972static bool hasFPCMov(unsigned X86CC) { 1973 switch (X86CC) { 1974 default: 1975 return false; 1976 case X86::COND_B: 1977 case X86::COND_BE: 1978 case X86::COND_E: 1979 case X86::COND_P: 1980 case X86::COND_A: 1981 case X86::COND_AE: 1982 case X86::COND_NE: 1983 case X86::COND_NP: 1984 return true; 1985 } 1986} 1987 1988/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 1989/// true if Op is undef or if its value falls within the specified range (L, H]. 1990static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) { 1991 if (Op.getOpcode() == ISD::UNDEF) 1992 return true; 1993 1994 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 1995 return (Val >= Low && Val < Hi); 1996} 1997 1998/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 1999/// true if Op is undef or if its value equal to the specified value. 2000static bool isUndefOrEqual(SDOperand Op, unsigned Val) { 2001 if (Op.getOpcode() == ISD::UNDEF) 2002 return true; 2003 return cast<ConstantSDNode>(Op)->getValue() == Val; 2004} 2005 2006/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2007/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2008bool X86::isPSHUFDMask(SDNode *N) { 2009 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2010 2011 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2012 return false; 2013 2014 // Check if the value doesn't reference the second vector. 2015 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2016 SDOperand Arg = N->getOperand(i); 2017 if (Arg.getOpcode() == ISD::UNDEF) continue; 2018 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2019 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2020 return false; 2021 } 2022 2023 return true; 2024} 2025 2026/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2027/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2028bool X86::isPSHUFHWMask(SDNode *N) { 2029 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2030 2031 if (N->getNumOperands() != 8) 2032 return false; 2033 2034 // Lower quadword copied in order. 2035 for (unsigned i = 0; i != 4; ++i) { 2036 SDOperand Arg = N->getOperand(i); 2037 if (Arg.getOpcode() == ISD::UNDEF) continue; 2038 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2039 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2040 return false; 2041 } 2042 2043 // Upper quadword shuffled. 2044 for (unsigned i = 4; i != 8; ++i) { 2045 SDOperand Arg = N->getOperand(i); 2046 if (Arg.getOpcode() == ISD::UNDEF) continue; 2047 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2048 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2049 if (Val < 4 || Val > 7) 2050 return false; 2051 } 2052 2053 return true; 2054} 2055 2056/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2057/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2058bool X86::isPSHUFLWMask(SDNode *N) { 2059 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2060 2061 if (N->getNumOperands() != 8) 2062 return false; 2063 2064 // Upper quadword copied in order. 2065 for (unsigned i = 4; i != 8; ++i) 2066 if (!isUndefOrEqual(N->getOperand(i), i)) 2067 return false; 2068 2069 // Lower quadword shuffled. 2070 for (unsigned i = 0; i != 4; ++i) 2071 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2072 return false; 2073 2074 return true; 2075} 2076 2077/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2078/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2079static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2080 if (NumElems != 2 && NumElems != 4) return false; 2081 2082 unsigned Half = NumElems / 2; 2083 for (unsigned i = 0; i < Half; ++i) 2084 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2085 return false; 2086 for (unsigned i = Half; i < NumElems; ++i) 2087 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2088 return false; 2089 2090 return true; 2091} 2092 2093bool X86::isSHUFPMask(SDNode *N) { 2094 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2095 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2096} 2097 2098/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2099/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2100/// half elements to come from vector 1 (which would equal the dest.) and 2101/// the upper half to come from vector 2. 2102static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2103 if (NumOps != 2 && NumOps != 4) return false; 2104 2105 unsigned Half = NumOps / 2; 2106 for (unsigned i = 0; i < Half; ++i) 2107 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2108 return false; 2109 for (unsigned i = Half; i < NumOps; ++i) 2110 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2111 return false; 2112 return true; 2113} 2114 2115static bool isCommutedSHUFP(SDNode *N) { 2116 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2117 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2118} 2119 2120/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2121/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2122bool X86::isMOVHLPSMask(SDNode *N) { 2123 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2124 2125 if (N->getNumOperands() != 4) 2126 return false; 2127 2128 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2129 return isUndefOrEqual(N->getOperand(0), 6) && 2130 isUndefOrEqual(N->getOperand(1), 7) && 2131 isUndefOrEqual(N->getOperand(2), 2) && 2132 isUndefOrEqual(N->getOperand(3), 3); 2133} 2134 2135/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2136/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2137/// <2, 3, 2, 3> 2138bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2139 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2140 2141 if (N->getNumOperands() != 4) 2142 return false; 2143 2144 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2145 return isUndefOrEqual(N->getOperand(0), 2) && 2146 isUndefOrEqual(N->getOperand(1), 3) && 2147 isUndefOrEqual(N->getOperand(2), 2) && 2148 isUndefOrEqual(N->getOperand(3), 3); 2149} 2150 2151/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2152/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2153bool X86::isMOVLPMask(SDNode *N) { 2154 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2155 2156 unsigned NumElems = N->getNumOperands(); 2157 if (NumElems != 2 && NumElems != 4) 2158 return false; 2159 2160 for (unsigned i = 0; i < NumElems/2; ++i) 2161 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2162 return false; 2163 2164 for (unsigned i = NumElems/2; i < NumElems; ++i) 2165 if (!isUndefOrEqual(N->getOperand(i), i)) 2166 return false; 2167 2168 return true; 2169} 2170 2171/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2172/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2173/// and MOVLHPS. 2174bool X86::isMOVHPMask(SDNode *N) { 2175 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2176 2177 unsigned NumElems = N->getNumOperands(); 2178 if (NumElems != 2 && NumElems != 4) 2179 return false; 2180 2181 for (unsigned i = 0; i < NumElems/2; ++i) 2182 if (!isUndefOrEqual(N->getOperand(i), i)) 2183 return false; 2184 2185 for (unsigned i = 0; i < NumElems/2; ++i) { 2186 SDOperand Arg = N->getOperand(i + NumElems/2); 2187 if (!isUndefOrEqual(Arg, i + NumElems)) 2188 return false; 2189 } 2190 2191 return true; 2192} 2193 2194/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2195/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2196bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2197 bool V2IsSplat = false) { 2198 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2199 return false; 2200 2201 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2202 SDOperand BitI = Elts[i]; 2203 SDOperand BitI1 = Elts[i+1]; 2204 if (!isUndefOrEqual(BitI, j)) 2205 return false; 2206 if (V2IsSplat) { 2207 if (isUndefOrEqual(BitI1, NumElts)) 2208 return false; 2209 } else { 2210 if (!isUndefOrEqual(BitI1, j + NumElts)) 2211 return false; 2212 } 2213 } 2214 2215 return true; 2216} 2217 2218bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2219 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2220 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2221} 2222 2223/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2224/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2225bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2226 bool V2IsSplat = false) { 2227 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2228 return false; 2229 2230 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2231 SDOperand BitI = Elts[i]; 2232 SDOperand BitI1 = Elts[i+1]; 2233 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2234 return false; 2235 if (V2IsSplat) { 2236 if (isUndefOrEqual(BitI1, NumElts)) 2237 return false; 2238 } else { 2239 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2240 return false; 2241 } 2242 } 2243 2244 return true; 2245} 2246 2247bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2248 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2249 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2250} 2251 2252/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2253/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2254/// <0, 0, 1, 1> 2255bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2256 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2257 2258 unsigned NumElems = N->getNumOperands(); 2259 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2260 return false; 2261 2262 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2263 SDOperand BitI = N->getOperand(i); 2264 SDOperand BitI1 = N->getOperand(i+1); 2265 2266 if (!isUndefOrEqual(BitI, j)) 2267 return false; 2268 if (!isUndefOrEqual(BitI1, j)) 2269 return false; 2270 } 2271 2272 return true; 2273} 2274 2275/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2276/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2277/// <2, 2, 3, 3> 2278bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2279 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2280 2281 unsigned NumElems = N->getNumOperands(); 2282 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2283 return false; 2284 2285 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2286 SDOperand BitI = N->getOperand(i); 2287 SDOperand BitI1 = N->getOperand(i + 1); 2288 2289 if (!isUndefOrEqual(BitI, j)) 2290 return false; 2291 if (!isUndefOrEqual(BitI1, j)) 2292 return false; 2293 } 2294 2295 return true; 2296} 2297 2298/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2299/// specifies a shuffle of elements that is suitable for input to MOVSS, 2300/// MOVSD, and MOVD, i.e. setting the lowest element. 2301static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2302 if (NumElts != 2 && NumElts != 4) 2303 return false; 2304 2305 if (!isUndefOrEqual(Elts[0], NumElts)) 2306 return false; 2307 2308 for (unsigned i = 1; i < NumElts; ++i) { 2309 if (!isUndefOrEqual(Elts[i], i)) 2310 return false; 2311 } 2312 2313 return true; 2314} 2315 2316bool X86::isMOVLMask(SDNode *N) { 2317 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2318 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2319} 2320 2321/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2322/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2323/// element of vector 2 and the other elements to come from vector 1 in order. 2324static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2325 bool V2IsSplat = false, 2326 bool V2IsUndef = false) { 2327 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2328 return false; 2329 2330 if (!isUndefOrEqual(Ops[0], 0)) 2331 return false; 2332 2333 for (unsigned i = 1; i < NumOps; ++i) { 2334 SDOperand Arg = Ops[i]; 2335 if (!(isUndefOrEqual(Arg, i+NumOps) || 2336 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2337 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2338 return false; 2339 } 2340 2341 return true; 2342} 2343 2344static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2345 bool V2IsUndef = false) { 2346 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2347 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2348 V2IsSplat, V2IsUndef); 2349} 2350 2351/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2352/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2353bool X86::isMOVSHDUPMask(SDNode *N) { 2354 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2355 2356 if (N->getNumOperands() != 4) 2357 return false; 2358 2359 // Expect 1, 1, 3, 3 2360 for (unsigned i = 0; i < 2; ++i) { 2361 SDOperand Arg = N->getOperand(i); 2362 if (Arg.getOpcode() == ISD::UNDEF) continue; 2363 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2364 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2365 if (Val != 1) return false; 2366 } 2367 2368 bool HasHi = false; 2369 for (unsigned i = 2; i < 4; ++i) { 2370 SDOperand Arg = N->getOperand(i); 2371 if (Arg.getOpcode() == ISD::UNDEF) continue; 2372 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2373 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2374 if (Val != 3) return false; 2375 HasHi = true; 2376 } 2377 2378 // Don't use movshdup if it can be done with a shufps. 2379 return HasHi; 2380} 2381 2382/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2383/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2384bool X86::isMOVSLDUPMask(SDNode *N) { 2385 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2386 2387 if (N->getNumOperands() != 4) 2388 return false; 2389 2390 // Expect 0, 0, 2, 2 2391 for (unsigned i = 0; i < 2; ++i) { 2392 SDOperand Arg = N->getOperand(i); 2393 if (Arg.getOpcode() == ISD::UNDEF) continue; 2394 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2395 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2396 if (Val != 0) return false; 2397 } 2398 2399 bool HasHi = false; 2400 for (unsigned i = 2; i < 4; ++i) { 2401 SDOperand Arg = N->getOperand(i); 2402 if (Arg.getOpcode() == ISD::UNDEF) continue; 2403 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2404 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2405 if (Val != 2) return false; 2406 HasHi = true; 2407 } 2408 2409 // Don't use movshdup if it can be done with a shufps. 2410 return HasHi; 2411} 2412 2413/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2414/// specifies a identity operation on the LHS or RHS. 2415static bool isIdentityMask(SDNode *N, bool RHS = false) { 2416 unsigned NumElems = N->getNumOperands(); 2417 for (unsigned i = 0; i < NumElems; ++i) 2418 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2419 return false; 2420 return true; 2421} 2422 2423/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2424/// a splat of a single element. 2425static bool isSplatMask(SDNode *N) { 2426 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2427 2428 // This is a splat operation if each element of the permute is the same, and 2429 // if the value doesn't reference the second vector. 2430 unsigned NumElems = N->getNumOperands(); 2431 SDOperand ElementBase; 2432 unsigned i = 0; 2433 for (; i != NumElems; ++i) { 2434 SDOperand Elt = N->getOperand(i); 2435 if (isa<ConstantSDNode>(Elt)) { 2436 ElementBase = Elt; 2437 break; 2438 } 2439 } 2440 2441 if (!ElementBase.Val) 2442 return false; 2443 2444 for (; i != NumElems; ++i) { 2445 SDOperand Arg = N->getOperand(i); 2446 if (Arg.getOpcode() == ISD::UNDEF) continue; 2447 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2448 if (Arg != ElementBase) return false; 2449 } 2450 2451 // Make sure it is a splat of the first vector operand. 2452 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2453} 2454 2455/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2456/// a splat of a single element and it's a 2 or 4 element mask. 2457bool X86::isSplatMask(SDNode *N) { 2458 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2459 2460 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2461 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2462 return false; 2463 return ::isSplatMask(N); 2464} 2465 2466/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2467/// specifies a splat of zero element. 2468bool X86::isSplatLoMask(SDNode *N) { 2469 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2470 2471 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2472 if (!isUndefOrEqual(N->getOperand(i), 0)) 2473 return false; 2474 return true; 2475} 2476 2477/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2478/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2479/// instructions. 2480unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2481 unsigned NumOperands = N->getNumOperands(); 2482 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2483 unsigned Mask = 0; 2484 for (unsigned i = 0; i < NumOperands; ++i) { 2485 unsigned Val = 0; 2486 SDOperand Arg = N->getOperand(NumOperands-i-1); 2487 if (Arg.getOpcode() != ISD::UNDEF) 2488 Val = cast<ConstantSDNode>(Arg)->getValue(); 2489 if (Val >= NumOperands) Val -= NumOperands; 2490 Mask |= Val; 2491 if (i != NumOperands - 1) 2492 Mask <<= Shift; 2493 } 2494 2495 return Mask; 2496} 2497 2498/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2499/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2500/// instructions. 2501unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2502 unsigned Mask = 0; 2503 // 8 nodes, but we only care about the last 4. 2504 for (unsigned i = 7; i >= 4; --i) { 2505 unsigned Val = 0; 2506 SDOperand Arg = N->getOperand(i); 2507 if (Arg.getOpcode() != ISD::UNDEF) 2508 Val = cast<ConstantSDNode>(Arg)->getValue(); 2509 Mask |= (Val - 4); 2510 if (i != 4) 2511 Mask <<= 2; 2512 } 2513 2514 return Mask; 2515} 2516 2517/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2518/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2519/// instructions. 2520unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2521 unsigned Mask = 0; 2522 // 8 nodes, but we only care about the first 4. 2523 for (int i = 3; i >= 0; --i) { 2524 unsigned Val = 0; 2525 SDOperand Arg = N->getOperand(i); 2526 if (Arg.getOpcode() != ISD::UNDEF) 2527 Val = cast<ConstantSDNode>(Arg)->getValue(); 2528 Mask |= Val; 2529 if (i != 0) 2530 Mask <<= 2; 2531 } 2532 2533 return Mask; 2534} 2535 2536/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2537/// specifies a 8 element shuffle that can be broken into a pair of 2538/// PSHUFHW and PSHUFLW. 2539static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2540 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2541 2542 if (N->getNumOperands() != 8) 2543 return false; 2544 2545 // Lower quadword shuffled. 2546 for (unsigned i = 0; i != 4; ++i) { 2547 SDOperand Arg = N->getOperand(i); 2548 if (Arg.getOpcode() == ISD::UNDEF) continue; 2549 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2550 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2551 if (Val >= 4) 2552 return false; 2553 } 2554 2555 // Upper quadword shuffled. 2556 for (unsigned i = 4; i != 8; ++i) { 2557 SDOperand Arg = N->getOperand(i); 2558 if (Arg.getOpcode() == ISD::UNDEF) continue; 2559 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2560 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2561 if (Val < 4 || Val > 7) 2562 return false; 2563 } 2564 2565 return true; 2566} 2567 2568/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2569/// values in ther permute mask. 2570static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1, 2571 SDOperand &V2, SDOperand &Mask, 2572 SelectionDAG &DAG) { 2573 MVT VT = Op.getValueType(); 2574 MVT MaskVT = Mask.getValueType(); 2575 MVT EltVT = MaskVT.getVectorElementType(); 2576 unsigned NumElems = Mask.getNumOperands(); 2577 SmallVector<SDOperand, 8> MaskVec; 2578 2579 for (unsigned i = 0; i != NumElems; ++i) { 2580 SDOperand Arg = Mask.getOperand(i); 2581 if (Arg.getOpcode() == ISD::UNDEF) { 2582 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2583 continue; 2584 } 2585 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2586 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2587 if (Val < NumElems) 2588 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2589 else 2590 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2591 } 2592 2593 std::swap(V1, V2); 2594 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2595 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2596} 2597 2598/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2599/// the two vector operands have swapped position. 2600static 2601SDOperand CommuteVectorShuffleMask(SDOperand Mask, SelectionDAG &DAG) { 2602 MVT MaskVT = Mask.getValueType(); 2603 MVT EltVT = MaskVT.getVectorElementType(); 2604 unsigned NumElems = Mask.getNumOperands(); 2605 SmallVector<SDOperand, 8> MaskVec; 2606 for (unsigned i = 0; i != NumElems; ++i) { 2607 SDOperand Arg = Mask.getOperand(i); 2608 if (Arg.getOpcode() == ISD::UNDEF) { 2609 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2610 continue; 2611 } 2612 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2613 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2614 if (Val < NumElems) 2615 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2616 else 2617 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2618 } 2619 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2620} 2621 2622 2623/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2624/// match movhlps. The lower half elements should come from upper half of 2625/// V1 (and in order), and the upper half elements should come from the upper 2626/// half of V2 (and in order). 2627static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2628 unsigned NumElems = Mask->getNumOperands(); 2629 if (NumElems != 4) 2630 return false; 2631 for (unsigned i = 0, e = 2; i != e; ++i) 2632 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2633 return false; 2634 for (unsigned i = 2; i != 4; ++i) 2635 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2636 return false; 2637 return true; 2638} 2639 2640/// isScalarLoadToVector - Returns true if the node is a scalar load that 2641/// is promoted to a vector. It also returns the LoadSDNode by reference if 2642/// required. 2643static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2644 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2645 N = N->getOperand(0).Val; 2646 if (ISD::isNON_EXTLoad(N)) { 2647 if (LD) 2648 *LD = cast<LoadSDNode>(N); 2649 return true; 2650 } 2651 } 2652 return false; 2653} 2654 2655/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2656/// match movlp{s|d}. The lower half elements should come from lower half of 2657/// V1 (and in order), and the upper half elements should come from the upper 2658/// half of V2 (and in order). And since V1 will become the source of the 2659/// MOVLP, it must be either a vector load or a scalar load to vector. 2660static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2661 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2662 return false; 2663 // Is V2 is a vector load, don't do this transformation. We will try to use 2664 // load folding shufps op. 2665 if (ISD::isNON_EXTLoad(V2)) 2666 return false; 2667 2668 unsigned NumElems = Mask->getNumOperands(); 2669 if (NumElems != 2 && NumElems != 4) 2670 return false; 2671 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2672 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2673 return false; 2674 for (unsigned i = NumElems/2; i != NumElems; ++i) 2675 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2676 return false; 2677 return true; 2678} 2679 2680/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2681/// all the same. 2682static bool isSplatVector(SDNode *N) { 2683 if (N->getOpcode() != ISD::BUILD_VECTOR) 2684 return false; 2685 2686 SDOperand SplatValue = N->getOperand(0); 2687 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2688 if (N->getOperand(i) != SplatValue) 2689 return false; 2690 return true; 2691} 2692 2693/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2694/// to an undef. 2695static bool isUndefShuffle(SDNode *N) { 2696 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2697 return false; 2698 2699 SDOperand V1 = N->getOperand(0); 2700 SDOperand V2 = N->getOperand(1); 2701 SDOperand Mask = N->getOperand(2); 2702 unsigned NumElems = Mask.getNumOperands(); 2703 for (unsigned i = 0; i != NumElems; ++i) { 2704 SDOperand Arg = Mask.getOperand(i); 2705 if (Arg.getOpcode() != ISD::UNDEF) { 2706 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2707 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2708 return false; 2709 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2710 return false; 2711 } 2712 } 2713 return true; 2714} 2715 2716/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2717/// constant +0.0. 2718static inline bool isZeroNode(SDOperand Elt) { 2719 return ((isa<ConstantSDNode>(Elt) && 2720 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2721 (isa<ConstantFPSDNode>(Elt) && 2722 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2723} 2724 2725/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2726/// to an zero vector. 2727static bool isZeroShuffle(SDNode *N) { 2728 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2729 return false; 2730 2731 SDOperand V1 = N->getOperand(0); 2732 SDOperand V2 = N->getOperand(1); 2733 SDOperand Mask = N->getOperand(2); 2734 unsigned NumElems = Mask.getNumOperands(); 2735 for (unsigned i = 0; i != NumElems; ++i) { 2736 SDOperand Arg = Mask.getOperand(i); 2737 if (Arg.getOpcode() == ISD::UNDEF) 2738 continue; 2739 2740 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2741 if (Idx < NumElems) { 2742 unsigned Opc = V1.Val->getOpcode(); 2743 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.Val)) 2744 continue; 2745 if (Opc != ISD::BUILD_VECTOR || 2746 !isZeroNode(V1.Val->getOperand(Idx))) 2747 return false; 2748 } else if (Idx >= NumElems) { 2749 unsigned Opc = V2.Val->getOpcode(); 2750 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.Val)) 2751 continue; 2752 if (Opc != ISD::BUILD_VECTOR || 2753 !isZeroNode(V2.Val->getOperand(Idx - NumElems))) 2754 return false; 2755 } 2756 } 2757 return true; 2758} 2759 2760/// getZeroVector - Returns a vector of specified type with all zero elements. 2761/// 2762static SDOperand getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2763 assert(VT.isVector() && "Expected a vector type"); 2764 2765 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2766 // type. This ensures they get CSE'd. 2767 SDOperand Vec; 2768 if (VT.getSizeInBits() == 64) { // MMX 2769 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2770 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2771 } else if (HasSSE2) { // SSE2 2772 SDOperand Cst = DAG.getTargetConstant(0, MVT::i32); 2773 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2774 } else { // SSE1 2775 SDOperand Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2776 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2777 } 2778 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2779} 2780 2781/// getOnesVector - Returns a vector of specified type with all bits set. 2782/// 2783static SDOperand getOnesVector(MVT VT, SelectionDAG &DAG) { 2784 assert(VT.isVector() && "Expected a vector type"); 2785 2786 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2787 // type. This ensures they get CSE'd. 2788 SDOperand Cst = DAG.getTargetConstant(~0U, MVT::i32); 2789 SDOperand Vec; 2790 if (VT.getSizeInBits() == 64) // MMX 2791 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2792 else // SSE 2793 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2794 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2795} 2796 2797 2798/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2799/// that point to V2 points to its first element. 2800static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) { 2801 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2802 2803 bool Changed = false; 2804 SmallVector<SDOperand, 8> MaskVec; 2805 unsigned NumElems = Mask.getNumOperands(); 2806 for (unsigned i = 0; i != NumElems; ++i) { 2807 SDOperand Arg = Mask.getOperand(i); 2808 if (Arg.getOpcode() != ISD::UNDEF) { 2809 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2810 if (Val > NumElems) { 2811 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2812 Changed = true; 2813 } 2814 } 2815 MaskVec.push_back(Arg); 2816 } 2817 2818 if (Changed) 2819 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2820 &MaskVec[0], MaskVec.size()); 2821 return Mask; 2822} 2823 2824/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2825/// operation of specified width. 2826static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2827 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2828 MVT BaseVT = MaskVT.getVectorElementType(); 2829 2830 SmallVector<SDOperand, 8> MaskVec; 2831 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2832 for (unsigned i = 1; i != NumElems; ++i) 2833 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2834 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2835} 2836 2837/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2838/// of specified width. 2839static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2840 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2841 MVT BaseVT = MaskVT.getVectorElementType(); 2842 SmallVector<SDOperand, 8> MaskVec; 2843 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2844 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2845 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2846 } 2847 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2848} 2849 2850/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2851/// of specified width. 2852static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2853 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2854 MVT BaseVT = MaskVT.getVectorElementType(); 2855 unsigned Half = NumElems/2; 2856 SmallVector<SDOperand, 8> MaskVec; 2857 for (unsigned i = 0; i != Half; ++i) { 2858 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2859 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2860 } 2861 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2862} 2863 2864/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2865/// element #0 of a vector with the specified index, leaving the rest of the 2866/// elements in place. 2867static SDOperand getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2868 SelectionDAG &DAG) { 2869 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2870 MVT BaseVT = MaskVT.getVectorElementType(); 2871 SmallVector<SDOperand, 8> MaskVec; 2872 // Element #0 of the result gets the elt we are replacing. 2873 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2874 for (unsigned i = 1; i != NumElems; ++i) 2875 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2876 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2877} 2878 2879/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2880static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) { 2881 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2882 MVT VT = Op.getValueType(); 2883 if (PVT == VT) 2884 return Op; 2885 SDOperand V1 = Op.getOperand(0); 2886 SDOperand Mask = Op.getOperand(2); 2887 unsigned NumElems = Mask.getNumOperands(); 2888 // Special handling of v4f32 -> v4i32. 2889 if (VT != MVT::v4f32) { 2890 Mask = getUnpacklMask(NumElems, DAG); 2891 while (NumElems > 4) { 2892 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2893 NumElems >>= 1; 2894 } 2895 Mask = getZeroVector(MVT::v4i32, true, DAG); 2896 } 2897 2898 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2899 SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2900 DAG.getNode(ISD::UNDEF, PVT), Mask); 2901 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2902} 2903 2904/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2905/// vector of zero or undef vector. This produces a shuffle where the low 2906/// element of V2 is swizzled into the zero/undef vector, landing at element 2907/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2908static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx, 2909 bool isZero, bool HasSSE2, 2910 SelectionDAG &DAG) { 2911 MVT VT = V2.getValueType(); 2912 SDOperand V1 = isZero 2913 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2914 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2915 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2916 MVT EVT = MaskVT.getVectorElementType(); 2917 SmallVector<SDOperand, 16> MaskVec; 2918 for (unsigned i = 0; i != NumElems; ++i) 2919 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2920 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2921 else 2922 MaskVec.push_back(DAG.getConstant(i, EVT)); 2923 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2924 &MaskVec[0], MaskVec.size()); 2925 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2926} 2927 2928/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2929/// a shuffle that is zero. 2930static 2931unsigned getNumOfConsecutiveZeros(SDOperand Op, SDOperand Mask, 2932 unsigned NumElems, bool Low, 2933 SelectionDAG &DAG) { 2934 unsigned NumZeros = 0; 2935 for (unsigned i = 0; i < NumElems; ++i) { 2936 SDOperand Idx = Mask.getOperand(Low ? i : NumElems-i-1); 2937 if (Idx.getOpcode() == ISD::UNDEF) { 2938 ++NumZeros; 2939 continue; 2940 } 2941 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 2942 SDOperand Elt = DAG.getShuffleScalarElt(Op.Val, Index); 2943 if (Elt.Val && isZeroNode(Elt)) 2944 ++NumZeros; 2945 else 2946 break; 2947 } 2948 return NumZeros; 2949} 2950 2951/// isVectorShift - Returns true if the shuffle can be implemented as a 2952/// logical left or right shift of a vector. 2953static bool isVectorShift(SDOperand Op, SDOperand Mask, SelectionDAG &DAG, 2954 bool &isLeft, SDOperand &ShVal, unsigned &ShAmt) { 2955 unsigned NumElems = Mask.getNumOperands(); 2956 2957 isLeft = true; 2958 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 2959 if (!NumZeros) { 2960 isLeft = false; 2961 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 2962 if (!NumZeros) 2963 return false; 2964 } 2965 2966 bool SeenV1 = false; 2967 bool SeenV2 = false; 2968 for (unsigned i = NumZeros; i < NumElems; ++i) { 2969 unsigned Val = isLeft ? (i - NumZeros) : i; 2970 SDOperand Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 2971 if (Idx.getOpcode() == ISD::UNDEF) 2972 continue; 2973 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 2974 if (Index < NumElems) 2975 SeenV1 = true; 2976 else { 2977 Index -= NumElems; 2978 SeenV2 = true; 2979 } 2980 if (Index != Val) 2981 return false; 2982 } 2983 if (SeenV1 && SeenV2) 2984 return false; 2985 2986 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 2987 ShAmt = NumZeros; 2988 return true; 2989} 2990 2991 2992/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 2993/// 2994static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros, 2995 unsigned NumNonZero, unsigned NumZero, 2996 SelectionDAG &DAG, TargetLowering &TLI) { 2997 if (NumNonZero > 8) 2998 return SDOperand(); 2999 3000 SDOperand V(0, 0); 3001 bool First = true; 3002 for (unsigned i = 0; i < 16; ++i) { 3003 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3004 if (ThisIsNonZero && First) { 3005 if (NumZero) 3006 V = getZeroVector(MVT::v8i16, true, DAG); 3007 else 3008 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3009 First = false; 3010 } 3011 3012 if ((i & 1) != 0) { 3013 SDOperand ThisElt(0, 0), LastElt(0, 0); 3014 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3015 if (LastIsNonZero) { 3016 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3017 } 3018 if (ThisIsNonZero) { 3019 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3020 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3021 ThisElt, DAG.getConstant(8, MVT::i8)); 3022 if (LastIsNonZero) 3023 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3024 } else 3025 ThisElt = LastElt; 3026 3027 if (ThisElt.Val) 3028 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3029 DAG.getIntPtrConstant(i/2)); 3030 } 3031 } 3032 3033 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3034} 3035 3036/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3037/// 3038static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros, 3039 unsigned NumNonZero, unsigned NumZero, 3040 SelectionDAG &DAG, TargetLowering &TLI) { 3041 if (NumNonZero > 4) 3042 return SDOperand(); 3043 3044 SDOperand V(0, 0); 3045 bool First = true; 3046 for (unsigned i = 0; i < 8; ++i) { 3047 bool isNonZero = (NonZeros & (1 << i)) != 0; 3048 if (isNonZero) { 3049 if (First) { 3050 if (NumZero) 3051 V = getZeroVector(MVT::v8i16, true, DAG); 3052 else 3053 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3054 First = false; 3055 } 3056 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3057 DAG.getIntPtrConstant(i)); 3058 } 3059 } 3060 3061 return V; 3062} 3063 3064/// getVShift - Return a vector logical shift node. 3065/// 3066static SDOperand getVShift(bool isLeft, MVT VT, SDOperand SrcOp, 3067 unsigned NumBits, SelectionDAG &DAG, 3068 const TargetLowering &TLI) { 3069 bool isMMX = VT.getSizeInBits() == 64; 3070 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3071 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3072 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3073 return DAG.getNode(ISD::BIT_CONVERT, VT, 3074 DAG.getNode(Opc, ShVT, SrcOp, 3075 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3076} 3077 3078SDOperand 3079X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) { 3080 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3081 if (ISD::isBuildVectorAllZeros(Op.Val) || ISD::isBuildVectorAllOnes(Op.Val)) { 3082 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3083 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3084 // eliminated on x86-32 hosts. 3085 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3086 return Op; 3087 3088 if (ISD::isBuildVectorAllOnes(Op.Val)) 3089 return getOnesVector(Op.getValueType(), DAG); 3090 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3091 } 3092 3093 MVT VT = Op.getValueType(); 3094 MVT EVT = VT.getVectorElementType(); 3095 unsigned EVTBits = EVT.getSizeInBits(); 3096 3097 unsigned NumElems = Op.getNumOperands(); 3098 unsigned NumZero = 0; 3099 unsigned NumNonZero = 0; 3100 unsigned NonZeros = 0; 3101 bool IsAllConstants = true; 3102 SmallSet<SDOperand, 8> Values; 3103 for (unsigned i = 0; i < NumElems; ++i) { 3104 SDOperand Elt = Op.getOperand(i); 3105 if (Elt.getOpcode() == ISD::UNDEF) 3106 continue; 3107 Values.insert(Elt); 3108 if (Elt.getOpcode() != ISD::Constant && 3109 Elt.getOpcode() != ISD::ConstantFP) 3110 IsAllConstants = false; 3111 if (isZeroNode(Elt)) 3112 NumZero++; 3113 else { 3114 NonZeros |= (1 << i); 3115 NumNonZero++; 3116 } 3117 } 3118 3119 if (NumNonZero == 0) { 3120 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3121 return DAG.getNode(ISD::UNDEF, VT); 3122 } 3123 3124 // Special case for single non-zero, non-undef, element. 3125 if (NumNonZero == 1 && NumElems <= 4) { 3126 unsigned Idx = CountTrailingZeros_32(NonZeros); 3127 SDOperand Item = Op.getOperand(Idx); 3128 3129 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3130 // the value are obviously zero, truncate the value to i32 and do the 3131 // insertion that way. Only do this if the value is non-constant or if the 3132 // value is a constant being inserted into element 0. It is cheaper to do 3133 // a constant pool load than it is to do a movd + shuffle. 3134 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3135 (!IsAllConstants || Idx == 0)) { 3136 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3137 // Handle MMX and SSE both. 3138 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3139 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3140 3141 // Truncate the value (which may itself be a constant) to i32, and 3142 // convert it to a vector with movd (S2V+shuffle to zero extend). 3143 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3144 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3145 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3146 Subtarget->hasSSE2(), DAG); 3147 3148 // Now we have our 32-bit value zero extended in the low element of 3149 // a vector. If Idx != 0, swizzle it into place. 3150 if (Idx != 0) { 3151 SDOperand Ops[] = { 3152 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3153 getSwapEltZeroMask(VecElts, Idx, DAG) 3154 }; 3155 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3156 } 3157 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3158 } 3159 } 3160 3161 // If we have a constant or non-constant insertion into the low element of 3162 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3163 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3164 // depending on what the source datatype is. Because we can only get here 3165 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3166 if (Idx == 0 && 3167 // Don't do this for i64 values on x86-32. 3168 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3169 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3170 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3171 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3172 Subtarget->hasSSE2(), DAG); 3173 } 3174 3175 // Is it a vector logical left shift? 3176 if (NumElems == 2 && Idx == 1 && 3177 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3178 unsigned NumBits = VT.getSizeInBits(); 3179 return getVShift(true, VT, 3180 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3181 NumBits/2, DAG, *this); 3182 } 3183 3184 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3185 return SDOperand(); 3186 3187 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3188 // is a non-constant being inserted into an element other than the low one, 3189 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3190 // movd/movss) to move this into the low element, then shuffle it into 3191 // place. 3192 if (EVTBits == 32) { 3193 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3194 3195 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3196 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3197 Subtarget->hasSSE2(), DAG); 3198 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3199 MVT MaskEVT = MaskVT.getVectorElementType(); 3200 SmallVector<SDOperand, 8> MaskVec; 3201 for (unsigned i = 0; i < NumElems; i++) 3202 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3203 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3204 &MaskVec[0], MaskVec.size()); 3205 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3206 DAG.getNode(ISD::UNDEF, VT), Mask); 3207 } 3208 } 3209 3210 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3211 if (Values.size() == 1) 3212 return SDOperand(); 3213 3214 // A vector full of immediates; various special cases are already 3215 // handled, so this is best done with a single constant-pool load. 3216 if (IsAllConstants) 3217 return SDOperand(); 3218 3219 // Let legalizer expand 2-wide build_vectors. 3220 if (EVTBits == 64) { 3221 if (NumNonZero == 1) { 3222 // One half is zero or undef. 3223 unsigned Idx = CountTrailingZeros_32(NonZeros); 3224 SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3225 Op.getOperand(Idx)); 3226 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3227 Subtarget->hasSSE2(), DAG); 3228 } 3229 return SDOperand(); 3230 } 3231 3232 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3233 if (EVTBits == 8 && NumElems == 16) { 3234 SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3235 *this); 3236 if (V.Val) return V; 3237 } 3238 3239 if (EVTBits == 16 && NumElems == 8) { 3240 SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3241 *this); 3242 if (V.Val) return V; 3243 } 3244 3245 // If element VT is == 32 bits, turn it into a number of shuffles. 3246 SmallVector<SDOperand, 8> V; 3247 V.resize(NumElems); 3248 if (NumElems == 4 && NumZero > 0) { 3249 for (unsigned i = 0; i < 4; ++i) { 3250 bool isZero = !(NonZeros & (1 << i)); 3251 if (isZero) 3252 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3253 else 3254 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3255 } 3256 3257 for (unsigned i = 0; i < 2; ++i) { 3258 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3259 default: break; 3260 case 0: 3261 V[i] = V[i*2]; // Must be a zero vector. 3262 break; 3263 case 1: 3264 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3265 getMOVLMask(NumElems, DAG)); 3266 break; 3267 case 2: 3268 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3269 getMOVLMask(NumElems, DAG)); 3270 break; 3271 case 3: 3272 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3273 getUnpacklMask(NumElems, DAG)); 3274 break; 3275 } 3276 } 3277 3278 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3279 MVT EVT = MaskVT.getVectorElementType(); 3280 SmallVector<SDOperand, 8> MaskVec; 3281 bool Reverse = (NonZeros & 0x3) == 2; 3282 for (unsigned i = 0; i < 2; ++i) 3283 if (Reverse) 3284 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3285 else 3286 MaskVec.push_back(DAG.getConstant(i, EVT)); 3287 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3288 for (unsigned i = 0; i < 2; ++i) 3289 if (Reverse) 3290 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3291 else 3292 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3293 SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3294 &MaskVec[0], MaskVec.size()); 3295 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3296 } 3297 3298 if (Values.size() > 2) { 3299 // Expand into a number of unpckl*. 3300 // e.g. for v4f32 3301 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3302 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3303 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3304 SDOperand UnpckMask = getUnpacklMask(NumElems, DAG); 3305 for (unsigned i = 0; i < NumElems; ++i) 3306 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3307 NumElems >>= 1; 3308 while (NumElems != 0) { 3309 for (unsigned i = 0; i < NumElems; ++i) 3310 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3311 UnpckMask); 3312 NumElems >>= 1; 3313 } 3314 return V[0]; 3315 } 3316 3317 return SDOperand(); 3318} 3319 3320static 3321SDOperand LowerVECTOR_SHUFFLEv8i16(SDOperand V1, SDOperand V2, 3322 SDOperand PermMask, SelectionDAG &DAG, 3323 TargetLowering &TLI) { 3324 SDOperand NewV; 3325 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3326 MVT MaskEVT = MaskVT.getVectorElementType(); 3327 MVT PtrVT = TLI.getPointerTy(); 3328 SmallVector<SDOperand, 8> MaskElts(PermMask.Val->op_begin(), 3329 PermMask.Val->op_end()); 3330 3331 // First record which half of which vector the low elements come from. 3332 SmallVector<unsigned, 4> LowQuad(4); 3333 for (unsigned i = 0; i < 4; ++i) { 3334 SDOperand Elt = MaskElts[i]; 3335 if (Elt.getOpcode() == ISD::UNDEF) 3336 continue; 3337 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3338 int QuadIdx = EltIdx / 4; 3339 ++LowQuad[QuadIdx]; 3340 } 3341 int BestLowQuad = -1; 3342 unsigned MaxQuad = 1; 3343 for (unsigned i = 0; i < 4; ++i) { 3344 if (LowQuad[i] > MaxQuad) { 3345 BestLowQuad = i; 3346 MaxQuad = LowQuad[i]; 3347 } 3348 } 3349 3350 // Record which half of which vector the high elements come from. 3351 SmallVector<unsigned, 4> HighQuad(4); 3352 for (unsigned i = 4; i < 8; ++i) { 3353 SDOperand Elt = MaskElts[i]; 3354 if (Elt.getOpcode() == ISD::UNDEF) 3355 continue; 3356 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3357 int QuadIdx = EltIdx / 4; 3358 ++HighQuad[QuadIdx]; 3359 } 3360 int BestHighQuad = -1; 3361 MaxQuad = 1; 3362 for (unsigned i = 0; i < 4; ++i) { 3363 if (HighQuad[i] > MaxQuad) { 3364 BestHighQuad = i; 3365 MaxQuad = HighQuad[i]; 3366 } 3367 } 3368 3369 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3370 if (BestLowQuad != -1 || BestHighQuad != -1) { 3371 // First sort the 4 chunks in order using shufpd. 3372 SmallVector<SDOperand, 8> MaskVec; 3373 if (BestLowQuad != -1) 3374 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3375 else 3376 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3377 if (BestHighQuad != -1) 3378 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3379 else 3380 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3381 SDOperand Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3382 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3383 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3384 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3385 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3386 3387 // Now sort high and low parts separately. 3388 BitVector InOrder(8); 3389 if (BestLowQuad != -1) { 3390 // Sort lower half in order using PSHUFLW. 3391 MaskVec.clear(); 3392 bool AnyOutOrder = false; 3393 for (unsigned i = 0; i != 4; ++i) { 3394 SDOperand Elt = MaskElts[i]; 3395 if (Elt.getOpcode() == ISD::UNDEF) { 3396 MaskVec.push_back(Elt); 3397 InOrder.set(i); 3398 } else { 3399 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3400 if (EltIdx != i) 3401 AnyOutOrder = true; 3402 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3403 // If this element is in the right place after this shuffle, then 3404 // remember it. 3405 if ((int)(EltIdx / 4) == BestLowQuad) 3406 InOrder.set(i); 3407 } 3408 } 3409 if (AnyOutOrder) { 3410 for (unsigned i = 4; i != 8; ++i) 3411 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3412 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3413 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3414 } 3415 } 3416 3417 if (BestHighQuad != -1) { 3418 // Sort high half in order using PSHUFHW if possible. 3419 MaskVec.clear(); 3420 for (unsigned i = 0; i != 4; ++i) 3421 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3422 bool AnyOutOrder = false; 3423 for (unsigned i = 4; i != 8; ++i) { 3424 SDOperand Elt = MaskElts[i]; 3425 if (Elt.getOpcode() == ISD::UNDEF) { 3426 MaskVec.push_back(Elt); 3427 InOrder.set(i); 3428 } else { 3429 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3430 if (EltIdx != i) 3431 AnyOutOrder = true; 3432 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3433 // If this element is in the right place after this shuffle, then 3434 // remember it. 3435 if ((int)(EltIdx / 4) == BestHighQuad) 3436 InOrder.set(i); 3437 } 3438 } 3439 if (AnyOutOrder) { 3440 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3441 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3442 } 3443 } 3444 3445 // The other elements are put in the right place using pextrw and pinsrw. 3446 for (unsigned i = 0; i != 8; ++i) { 3447 if (InOrder[i]) 3448 continue; 3449 SDOperand Elt = MaskElts[i]; 3450 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3451 SDOperand ExtOp = (EltIdx < 8) 3452 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3453 DAG.getConstant(EltIdx, PtrVT)) 3454 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3455 DAG.getConstant(EltIdx - 8, PtrVT)); 3456 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3457 DAG.getConstant(i, PtrVT)); 3458 } 3459 return NewV; 3460 } 3461 3462 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use 3463 ///as few as possible. 3464 // First, let's find out how many elements are already in the right order. 3465 unsigned V1InOrder = 0; 3466 unsigned V1FromV1 = 0; 3467 unsigned V2InOrder = 0; 3468 unsigned V2FromV2 = 0; 3469 SmallVector<SDOperand, 8> V1Elts; 3470 SmallVector<SDOperand, 8> V2Elts; 3471 for (unsigned i = 0; i < 8; ++i) { 3472 SDOperand Elt = MaskElts[i]; 3473 if (Elt.getOpcode() == ISD::UNDEF) { 3474 V1Elts.push_back(Elt); 3475 V2Elts.push_back(Elt); 3476 ++V1InOrder; 3477 ++V2InOrder; 3478 continue; 3479 } 3480 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3481 if (EltIdx == i) { 3482 V1Elts.push_back(Elt); 3483 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3484 ++V1InOrder; 3485 } else if (EltIdx == i+8) { 3486 V1Elts.push_back(Elt); 3487 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3488 ++V2InOrder; 3489 } else if (EltIdx < 8) { 3490 V1Elts.push_back(Elt); 3491 ++V1FromV1; 3492 } else { 3493 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3494 ++V2FromV2; 3495 } 3496 } 3497 3498 if (V2InOrder > V1InOrder) { 3499 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3500 std::swap(V1, V2); 3501 std::swap(V1Elts, V2Elts); 3502 std::swap(V1FromV1, V2FromV2); 3503 } 3504 3505 if ((V1FromV1 + V1InOrder) != 8) { 3506 // Some elements are from V2. 3507 if (V1FromV1) { 3508 // If there are elements that are from V1 but out of place, 3509 // then first sort them in place 3510 SmallVector<SDOperand, 8> MaskVec; 3511 for (unsigned i = 0; i < 8; ++i) { 3512 SDOperand Elt = V1Elts[i]; 3513 if (Elt.getOpcode() == ISD::UNDEF) { 3514 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3515 continue; 3516 } 3517 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3518 if (EltIdx >= 8) 3519 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3520 else 3521 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3522 } 3523 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3524 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3525 } 3526 3527 NewV = V1; 3528 for (unsigned i = 0; i < 8; ++i) { 3529 SDOperand Elt = V1Elts[i]; 3530 if (Elt.getOpcode() == ISD::UNDEF) 3531 continue; 3532 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3533 if (EltIdx < 8) 3534 continue; 3535 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3536 DAG.getConstant(EltIdx - 8, PtrVT)); 3537 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3538 DAG.getConstant(i, PtrVT)); 3539 } 3540 return NewV; 3541 } else { 3542 // All elements are from V1. 3543 NewV = V1; 3544 for (unsigned i = 0; i < 8; ++i) { 3545 SDOperand Elt = V1Elts[i]; 3546 if (Elt.getOpcode() == ISD::UNDEF) 3547 continue; 3548 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3549 SDOperand ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3550 DAG.getConstant(EltIdx, PtrVT)); 3551 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3552 DAG.getConstant(i, PtrVT)); 3553 } 3554 return NewV; 3555 } 3556} 3557 3558/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3559/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3560/// done when every pair / quad of shuffle mask elements point to elements in 3561/// the right sequence. e.g. 3562/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3563static 3564SDOperand RewriteAsNarrowerShuffle(SDOperand V1, SDOperand V2, 3565 MVT VT, 3566 SDOperand PermMask, SelectionDAG &DAG, 3567 TargetLowering &TLI) { 3568 unsigned NumElems = PermMask.getNumOperands(); 3569 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3570 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3571 MVT NewVT = MaskVT; 3572 switch (VT.getSimpleVT()) { 3573 default: assert(false && "Unexpected!"); 3574 case MVT::v4f32: NewVT = MVT::v2f64; break; 3575 case MVT::v4i32: NewVT = MVT::v2i64; break; 3576 case MVT::v8i16: NewVT = MVT::v4i32; break; 3577 case MVT::v16i8: NewVT = MVT::v4i32; break; 3578 } 3579 3580 if (NewWidth == 2) { 3581 if (VT.isInteger()) 3582 NewVT = MVT::v2i64; 3583 else 3584 NewVT = MVT::v2f64; 3585 } 3586 unsigned Scale = NumElems / NewWidth; 3587 SmallVector<SDOperand, 8> MaskVec; 3588 for (unsigned i = 0; i < NumElems; i += Scale) { 3589 unsigned StartIdx = ~0U; 3590 for (unsigned j = 0; j < Scale; ++j) { 3591 SDOperand Elt = PermMask.getOperand(i+j); 3592 if (Elt.getOpcode() == ISD::UNDEF) 3593 continue; 3594 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3595 if (StartIdx == ~0U) 3596 StartIdx = EltIdx - (EltIdx % Scale); 3597 if (EltIdx != StartIdx + j) 3598 return SDOperand(); 3599 } 3600 if (StartIdx == ~0U) 3601 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MVT::i32)); 3602 else 3603 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MVT::i32)); 3604 } 3605 3606 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3607 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3608 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3609 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3610 &MaskVec[0], MaskVec.size())); 3611} 3612 3613/// getVZextMovL - Return a zero-extending vector move low node. 3614/// 3615static SDOperand getVZextMovL(MVT VT, MVT OpVT, 3616 SDOperand SrcOp, SelectionDAG &DAG, 3617 const X86Subtarget *Subtarget) { 3618 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3619 LoadSDNode *LD = NULL; 3620 if (!isScalarLoadToVector(SrcOp.Val, &LD)) 3621 LD = dyn_cast<LoadSDNode>(SrcOp); 3622 if (!LD) { 3623 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3624 // instead. 3625 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3626 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3627 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3628 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3629 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3630 // PR2108 3631 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3632 return DAG.getNode(ISD::BIT_CONVERT, VT, 3633 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3634 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3635 SrcOp.getOperand(0).getOperand(0)))); 3636 } 3637 } 3638 } 3639 3640 return DAG.getNode(ISD::BIT_CONVERT, VT, 3641 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3642 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3643} 3644 3645SDOperand 3646X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) { 3647 SDOperand V1 = Op.getOperand(0); 3648 SDOperand V2 = Op.getOperand(1); 3649 SDOperand PermMask = Op.getOperand(2); 3650 MVT VT = Op.getValueType(); 3651 unsigned NumElems = PermMask.getNumOperands(); 3652 bool isMMX = VT.getSizeInBits() == 64; 3653 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3654 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3655 bool V1IsSplat = false; 3656 bool V2IsSplat = false; 3657 3658 if (isUndefShuffle(Op.Val)) 3659 return DAG.getNode(ISD::UNDEF, VT); 3660 3661 if (isZeroShuffle(Op.Val)) 3662 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3663 3664 if (isIdentityMask(PermMask.Val)) 3665 return V1; 3666 else if (isIdentityMask(PermMask.Val, true)) 3667 return V2; 3668 3669 if (isSplatMask(PermMask.Val)) { 3670 if (isMMX || NumElems < 4) return Op; 3671 // Promote it to a v4{if}32 splat. 3672 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3673 } 3674 3675 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3676 // do it! 3677 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3678 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3679 if (NewOp.Val) 3680 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3681 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3682 // FIXME: Figure out a cleaner way to do this. 3683 // Try to make use of movq to zero out the top part. 3684 if (ISD::isBuildVectorAllZeros(V2.Val)) { 3685 SDOperand NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3686 DAG, *this); 3687 if (NewOp.Val) { 3688 SDOperand NewV1 = NewOp.getOperand(0); 3689 SDOperand NewV2 = NewOp.getOperand(1); 3690 SDOperand NewMask = NewOp.getOperand(2); 3691 if (isCommutedMOVL(NewMask.Val, true, false)) { 3692 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3693 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3694 } 3695 } 3696 } else if (ISD::isBuildVectorAllZeros(V1.Val)) { 3697 SDOperand NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3698 DAG, *this); 3699 if (NewOp.Val && X86::isMOVLMask(NewOp.getOperand(2).Val)) 3700 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3701 DAG, Subtarget); 3702 } 3703 } 3704 3705 // Check if this can be converted into a logical shift. 3706 bool isLeft = false; 3707 unsigned ShAmt = 0; 3708 SDOperand ShVal; 3709 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3710 if (isShift && ShVal.hasOneUse()) { 3711 // If the shifted value has multiple uses, it may be cheaper to use 3712 // v_set0 + movlhps or movhlps, etc. 3713 MVT EVT = VT.getVectorElementType(); 3714 ShAmt *= EVT.getSizeInBits(); 3715 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3716 } 3717 3718 if (X86::isMOVLMask(PermMask.Val)) { 3719 if (V1IsUndef) 3720 return V2; 3721 if (ISD::isBuildVectorAllZeros(V1.Val)) 3722 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3723 return Op; 3724 } 3725 3726 if (X86::isMOVSHDUPMask(PermMask.Val) || 3727 X86::isMOVSLDUPMask(PermMask.Val) || 3728 X86::isMOVHLPSMask(PermMask.Val) || 3729 X86::isMOVHPMask(PermMask.Val) || 3730 X86::isMOVLPMask(PermMask.Val)) 3731 return Op; 3732 3733 if (ShouldXformToMOVHLPS(PermMask.Val) || 3734 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val)) 3735 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3736 3737 if (isShift) { 3738 // No better options. Use a vshl / vsrl. 3739 MVT EVT = VT.getVectorElementType(); 3740 ShAmt *= EVT.getSizeInBits(); 3741 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3742 } 3743 3744 bool Commuted = false; 3745 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3746 // 1,1,1,1 -> v8i16 though. 3747 V1IsSplat = isSplatVector(V1.Val); 3748 V2IsSplat = isSplatVector(V2.Val); 3749 3750 // Canonicalize the splat or undef, if present, to be on the RHS. 3751 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3752 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3753 std::swap(V1IsSplat, V2IsSplat); 3754 std::swap(V1IsUndef, V2IsUndef); 3755 Commuted = true; 3756 } 3757 3758 // FIXME: Figure out a cleaner way to do this. 3759 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) { 3760 if (V2IsUndef) return V1; 3761 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3762 if (V2IsSplat) { 3763 // V2 is a splat, so the mask may be malformed. That is, it may point 3764 // to any V2 element. The instruction selectior won't like this. Get 3765 // a corrected mask and commute to form a proper MOVS{S|D}. 3766 SDOperand NewMask = getMOVLMask(NumElems, DAG); 3767 if (NewMask.Val != PermMask.Val) 3768 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3769 } 3770 return Op; 3771 } 3772 3773 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3774 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3775 X86::isUNPCKLMask(PermMask.Val) || 3776 X86::isUNPCKHMask(PermMask.Val)) 3777 return Op; 3778 3779 if (V2IsSplat) { 3780 // Normalize mask so all entries that point to V2 points to its first 3781 // element then try to match unpck{h|l} again. If match, return a 3782 // new vector_shuffle with the corrected mask. 3783 SDOperand NewMask = NormalizeMask(PermMask, DAG); 3784 if (NewMask.Val != PermMask.Val) { 3785 if (X86::isUNPCKLMask(PermMask.Val, true)) { 3786 SDOperand NewMask = getUnpacklMask(NumElems, DAG); 3787 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3788 } else if (X86::isUNPCKHMask(PermMask.Val, true)) { 3789 SDOperand NewMask = getUnpackhMask(NumElems, DAG); 3790 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3791 } 3792 } 3793 } 3794 3795 // Normalize the node to match x86 shuffle ops if needed 3796 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val)) 3797 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3798 3799 if (Commuted) { 3800 // Commute is back and try unpck* again. 3801 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3802 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3803 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3804 X86::isUNPCKLMask(PermMask.Val) || 3805 X86::isUNPCKHMask(PermMask.Val)) 3806 return Op; 3807 } 3808 3809 // Try PSHUF* first, then SHUFP*. 3810 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 3811 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 3812 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.Val)) { 3813 if (V2.getOpcode() != ISD::UNDEF) 3814 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 3815 DAG.getNode(ISD::UNDEF, VT), PermMask); 3816 return Op; 3817 } 3818 3819 if (!isMMX) { 3820 if (Subtarget->hasSSE2() && 3821 (X86::isPSHUFDMask(PermMask.Val) || 3822 X86::isPSHUFHWMask(PermMask.Val) || 3823 X86::isPSHUFLWMask(PermMask.Val))) { 3824 MVT RVT = VT; 3825 if (VT == MVT::v4f32) { 3826 RVT = MVT::v4i32; 3827 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 3828 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 3829 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3830 } else if (V2.getOpcode() != ISD::UNDEF) 3831 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 3832 DAG.getNode(ISD::UNDEF, RVT), PermMask); 3833 if (RVT != VT) 3834 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 3835 return Op; 3836 } 3837 3838 // Binary or unary shufps. 3839 if (X86::isSHUFPMask(PermMask.Val) || 3840 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.Val))) 3841 return Op; 3842 } 3843 3844 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 3845 if (VT == MVT::v8i16) { 3846 SDOperand NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 3847 if (NewOp.Val) 3848 return NewOp; 3849 } 3850 3851 // Handle all 4 wide cases with a number of shuffles. 3852 if (NumElems == 4 && !isMMX) { 3853 // Don't do this for MMX. 3854 MVT MaskVT = PermMask.getValueType(); 3855 MVT MaskEVT = MaskVT.getVectorElementType(); 3856 SmallVector<std::pair<int, int>, 8> Locs; 3857 Locs.reserve(NumElems); 3858 SmallVector<SDOperand, 8> Mask1(NumElems, 3859 DAG.getNode(ISD::UNDEF, MaskEVT)); 3860 SmallVector<SDOperand, 8> Mask2(NumElems, 3861 DAG.getNode(ISD::UNDEF, MaskEVT)); 3862 unsigned NumHi = 0; 3863 unsigned NumLo = 0; 3864 // If no more than two elements come from either vector. This can be 3865 // implemented with two shuffles. First shuffle gather the elements. 3866 // The second shuffle, which takes the first shuffle as both of its 3867 // vector operands, put the elements into the right order. 3868 for (unsigned i = 0; i != NumElems; ++i) { 3869 SDOperand Elt = PermMask.getOperand(i); 3870 if (Elt.getOpcode() == ISD::UNDEF) { 3871 Locs[i] = std::make_pair(-1, -1); 3872 } else { 3873 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3874 if (Val < NumElems) { 3875 Locs[i] = std::make_pair(0, NumLo); 3876 Mask1[NumLo] = Elt; 3877 NumLo++; 3878 } else { 3879 Locs[i] = std::make_pair(1, NumHi); 3880 if (2+NumHi < NumElems) 3881 Mask1[2+NumHi] = Elt; 3882 NumHi++; 3883 } 3884 } 3885 } 3886 if (NumLo <= 2 && NumHi <= 2) { 3887 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3888 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3889 &Mask1[0], Mask1.size())); 3890 for (unsigned i = 0; i != NumElems; ++i) { 3891 if (Locs[i].first == -1) 3892 continue; 3893 else { 3894 unsigned Idx = (i < NumElems/2) ? 0 : NumElems; 3895 Idx += Locs[i].first * (NumElems/2) + Locs[i].second; 3896 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3897 } 3898 } 3899 3900 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3901 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3902 &Mask2[0], Mask2.size())); 3903 } 3904 3905 // Break it into (shuffle shuffle_hi, shuffle_lo). 3906 Locs.clear(); 3907 SmallVector<SDOperand,8> LoMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); 3908 SmallVector<SDOperand,8> HiMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); 3909 SmallVector<SDOperand,8> *MaskPtr = &LoMask; 3910 unsigned MaskIdx = 0; 3911 unsigned LoIdx = 0; 3912 unsigned HiIdx = NumElems/2; 3913 for (unsigned i = 0; i != NumElems; ++i) { 3914 if (i == NumElems/2) { 3915 MaskPtr = &HiMask; 3916 MaskIdx = 1; 3917 LoIdx = 0; 3918 HiIdx = NumElems/2; 3919 } 3920 SDOperand Elt = PermMask.getOperand(i); 3921 if (Elt.getOpcode() == ISD::UNDEF) { 3922 Locs[i] = std::make_pair(-1, -1); 3923 } else if (cast<ConstantSDNode>(Elt)->getValue() < NumElems) { 3924 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3925 (*MaskPtr)[LoIdx] = Elt; 3926 LoIdx++; 3927 } else { 3928 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3929 (*MaskPtr)[HiIdx] = Elt; 3930 HiIdx++; 3931 } 3932 } 3933 3934 SDOperand LoShuffle = 3935 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3936 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3937 &LoMask[0], LoMask.size())); 3938 SDOperand HiShuffle = 3939 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3940 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3941 &HiMask[0], HiMask.size())); 3942 SmallVector<SDOperand, 8> MaskOps; 3943 for (unsigned i = 0; i != NumElems; ++i) { 3944 if (Locs[i].first == -1) { 3945 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3946 } else { 3947 unsigned Idx = Locs[i].first * NumElems + Locs[i].second; 3948 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3949 } 3950 } 3951 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3952 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3953 &MaskOps[0], MaskOps.size())); 3954 } 3955 3956 return SDOperand(); 3957} 3958 3959SDOperand 3960X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDOperand Op, 3961 SelectionDAG &DAG) { 3962 MVT VT = Op.getValueType(); 3963 if (VT.getSizeInBits() == 8) { 3964 SDOperand Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 3965 Op.getOperand(0), Op.getOperand(1)); 3966 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 3967 DAG.getValueType(VT)); 3968 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 3969 } else if (VT.getSizeInBits() == 16) { 3970 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 3971 Op.getOperand(0), Op.getOperand(1)); 3972 SDOperand Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 3973 DAG.getValueType(VT)); 3974 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 3975 } else if (VT == MVT::f32) { 3976 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 3977 // the result back to FR32 register. It's only worth matching if the 3978 // result has a single use which is a store or a bitcast to i32. 3979 if (!Op.hasOneUse()) 3980 return SDOperand(); 3981 SDNode *User = Op.Val->use_begin()->getUser(); 3982 if (User->getOpcode() != ISD::STORE && 3983 (User->getOpcode() != ISD::BIT_CONVERT || 3984 User->getValueType(0) != MVT::i32)) 3985 return SDOperand(); 3986 SDOperand Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 3987 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 3988 Op.getOperand(1)); 3989 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 3990 } 3991 return SDOperand(); 3992} 3993 3994 3995SDOperand 3996X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 3997 if (!isa<ConstantSDNode>(Op.getOperand(1))) 3998 return SDOperand(); 3999 4000 if (Subtarget->hasSSE41()) { 4001 SDOperand Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 4002 if (Res.Val) 4003 return Res; 4004 } 4005 4006 MVT VT = Op.getValueType(); 4007 // TODO: handle v16i8. 4008 if (VT.getSizeInBits() == 16) { 4009 SDOperand Vec = Op.getOperand(0); 4010 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4011 if (Idx == 0) 4012 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4013 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4014 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4015 Op.getOperand(1))); 4016 // Transform it so it match pextrw which produces a 32-bit result. 4017 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4018 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4019 Op.getOperand(0), Op.getOperand(1)); 4020 SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4021 DAG.getValueType(VT)); 4022 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4023 } else if (VT.getSizeInBits() == 32) { 4024 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4025 if (Idx == 0) 4026 return Op; 4027 // SHUFPS the element to the lowest double word, then movss. 4028 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4029 SmallVector<SDOperand, 8> IdxVec; 4030 IdxVec. 4031 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4032 IdxVec. 4033 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4034 IdxVec. 4035 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4036 IdxVec. 4037 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4038 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4039 &IdxVec[0], IdxVec.size()); 4040 SDOperand Vec = Op.getOperand(0); 4041 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4042 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4043 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4044 DAG.getIntPtrConstant(0)); 4045 } else if (VT.getSizeInBits() == 64) { 4046 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4047 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4048 // to match extract_elt for f64. 4049 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4050 if (Idx == 0) 4051 return Op; 4052 4053 // UNPCKHPD the element to the lowest double word, then movsd. 4054 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4055 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4056 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4057 SmallVector<SDOperand, 8> IdxVec; 4058 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4059 IdxVec. 4060 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4061 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4062 &IdxVec[0], IdxVec.size()); 4063 SDOperand Vec = Op.getOperand(0); 4064 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4065 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4066 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4067 DAG.getIntPtrConstant(0)); 4068 } 4069 4070 return SDOperand(); 4071} 4072 4073SDOperand 4074X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDOperand Op, SelectionDAG &DAG){ 4075 MVT VT = Op.getValueType(); 4076 MVT EVT = VT.getVectorElementType(); 4077 4078 SDOperand N0 = Op.getOperand(0); 4079 SDOperand N1 = Op.getOperand(1); 4080 SDOperand N2 = Op.getOperand(2); 4081 4082 if ((EVT.getSizeInBits() == 8) || (EVT.getSizeInBits() == 16)) { 4083 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4084 : X86ISD::PINSRW; 4085 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4086 // argument. 4087 if (N1.getValueType() != MVT::i32) 4088 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4089 if (N2.getValueType() != MVT::i32) 4090 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4091 return DAG.getNode(Opc, VT, N0, N1, N2); 4092 } else if (EVT == MVT::f32) { 4093 // Bits [7:6] of the constant are the source select. This will always be 4094 // zero here. The DAG Combiner may combine an extract_elt index into these 4095 // bits. For example (insert (extract, 3), 2) could be matched by putting 4096 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4097 // Bits [5:4] of the constant are the destination select. This is the 4098 // value of the incoming immediate. 4099 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4100 // combine either bitwise AND or insert of float 0.0 to set these bits. 4101 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4102 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4103 } 4104 return SDOperand(); 4105} 4106 4107SDOperand 4108X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { 4109 MVT VT = Op.getValueType(); 4110 MVT EVT = VT.getVectorElementType(); 4111 4112 if (Subtarget->hasSSE41()) 4113 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4114 4115 if (EVT == MVT::i8) 4116 return SDOperand(); 4117 4118 SDOperand N0 = Op.getOperand(0); 4119 SDOperand N1 = Op.getOperand(1); 4120 SDOperand N2 = Op.getOperand(2); 4121 4122 if (EVT.getSizeInBits() == 16) { 4123 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4124 // as its second argument. 4125 if (N1.getValueType() != MVT::i32) 4126 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4127 if (N2.getValueType() != MVT::i32) 4128 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4129 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4130 } 4131 return SDOperand(); 4132} 4133 4134SDOperand 4135X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) { 4136 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4137 MVT VT = MVT::v2i32; 4138 switch (Op.getValueType().getSimpleVT()) { 4139 default: break; 4140 case MVT::v16i8: 4141 case MVT::v8i16: 4142 VT = MVT::v4i32; 4143 break; 4144 } 4145 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4146 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4147} 4148 4149// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4150// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4151// one of the above mentioned nodes. It has to be wrapped because otherwise 4152// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4153// be used to form addressing mode. These wrapped nodes will be selected 4154// into MOV32ri. 4155SDOperand 4156X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) { 4157 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4158 SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(), 4159 getPointerTy(), 4160 CP->getAlignment()); 4161 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4162 // With PIC, the address is actually $g + Offset. 4163 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4164 !Subtarget->isPICStyleRIPRel()) { 4165 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4166 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4167 Result); 4168 } 4169 4170 return Result; 4171} 4172 4173SDOperand 4174X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) { 4175 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4176 SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4177 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4178 // With PIC, the address is actually $g + Offset. 4179 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4180 !Subtarget->isPICStyleRIPRel()) { 4181 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4182 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4183 Result); 4184 } 4185 4186 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4187 // load the value at address GV, not the value of GV itself. This means that 4188 // the GlobalAddress must be in the base or index register of the address, not 4189 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4190 // The same applies for external symbols during PIC codegen 4191 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4192 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4193 PseudoSourceValue::getGOT(), 0); 4194 4195 return Result; 4196} 4197 4198// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4199static SDOperand 4200LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4201 const MVT PtrVT) { 4202 SDOperand InFlag; 4203 SDOperand Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4204 DAG.getNode(X86ISD::GlobalBaseReg, 4205 PtrVT), InFlag); 4206 InFlag = Chain.getValue(1); 4207 4208 // emit leal symbol@TLSGD(,%ebx,1), %eax 4209 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4210 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4211 GA->getValueType(0), 4212 GA->getOffset()); 4213 SDOperand Ops[] = { Chain, TGA, InFlag }; 4214 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4215 InFlag = Result.getValue(2); 4216 Chain = Result.getValue(1); 4217 4218 // call ___tls_get_addr. This function receives its argument in 4219 // the register EAX. 4220 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4221 InFlag = Chain.getValue(1); 4222 4223 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4224 SDOperand Ops1[] = { Chain, 4225 DAG.getTargetExternalSymbol("___tls_get_addr", 4226 PtrVT), 4227 DAG.getRegister(X86::EAX, PtrVT), 4228 DAG.getRegister(X86::EBX, PtrVT), 4229 InFlag }; 4230 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4231 InFlag = Chain.getValue(1); 4232 4233 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4234} 4235 4236// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4237static SDOperand 4238LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4239 const MVT PtrVT) { 4240 SDOperand InFlag, Chain; 4241 4242 // emit leaq symbol@TLSGD(%rip), %rdi 4243 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4244 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4245 GA->getValueType(0), 4246 GA->getOffset()); 4247 SDOperand Ops[] = { DAG.getEntryNode(), TGA}; 4248 SDOperand Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4249 Chain = Result.getValue(1); 4250 InFlag = Result.getValue(2); 4251 4252 // call ___tls_get_addr. This function receives its argument in 4253 // the register RDI. 4254 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4255 InFlag = Chain.getValue(1); 4256 4257 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4258 SDOperand Ops1[] = { Chain, 4259 DAG.getTargetExternalSymbol("___tls_get_addr", 4260 PtrVT), 4261 DAG.getRegister(X86::RDI, PtrVT), 4262 InFlag }; 4263 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4264 InFlag = Chain.getValue(1); 4265 4266 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4267} 4268 4269// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4270// "local exec" model. 4271static SDOperand LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4272 const MVT PtrVT) { 4273 // Get the Thread Pointer 4274 SDOperand ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4275 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4276 // exec) 4277 SDOperand TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4278 GA->getValueType(0), 4279 GA->getOffset()); 4280 SDOperand Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4281 4282 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4283 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4284 PseudoSourceValue::getGOT(), 0); 4285 4286 // The address of the thread local variable is the add of the thread 4287 // pointer with the offset of the variable. 4288 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4289} 4290 4291SDOperand 4292X86TargetLowering::LowerGlobalTLSAddress(SDOperand Op, SelectionDAG &DAG) { 4293 // TODO: implement the "local dynamic" model 4294 // TODO: implement the "initial exec"model for pic executables 4295 assert(Subtarget->isTargetELF() && 4296 "TLS not implemented for non-ELF targets"); 4297 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4298 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4299 // otherwise use the "Local Exec"TLS Model 4300 if (Subtarget->is64Bit()) { 4301 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4302 } else { 4303 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4304 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4305 else 4306 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4307 } 4308} 4309 4310SDOperand 4311X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) { 4312 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4313 SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4314 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4315 // With PIC, the address is actually $g + Offset. 4316 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4317 !Subtarget->isPICStyleRIPRel()) { 4318 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4319 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4320 Result); 4321 } 4322 4323 return Result; 4324} 4325 4326SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) { 4327 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4328 SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4329 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4330 // With PIC, the address is actually $g + Offset. 4331 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4332 !Subtarget->isPICStyleRIPRel()) { 4333 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4334 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4335 Result); 4336 } 4337 4338 return Result; 4339} 4340 4341/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4342/// take a 2 x i32 value to shift plus a shift amount. 4343SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) { 4344 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4345 MVT VT = Op.getValueType(); 4346 unsigned VTBits = VT.getSizeInBits(); 4347 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4348 SDOperand ShOpLo = Op.getOperand(0); 4349 SDOperand ShOpHi = Op.getOperand(1); 4350 SDOperand ShAmt = Op.getOperand(2); 4351 SDOperand Tmp1 = isSRA ? 4352 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4353 DAG.getConstant(0, VT); 4354 4355 SDOperand Tmp2, Tmp3; 4356 if (Op.getOpcode() == ISD::SHL_PARTS) { 4357 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4358 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4359 } else { 4360 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4361 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4362 } 4363 4364 const MVT *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); 4365 SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4366 DAG.getConstant(VTBits, MVT::i8)); 4367 SDOperand Cond = DAG.getNode(X86ISD::CMP, VT, 4368 AndNode, DAG.getConstant(0, MVT::i8)); 4369 4370 SDOperand Hi, Lo; 4371 SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4372 VTs = DAG.getNodeValueTypes(VT, MVT::Flag); 4373 SmallVector<SDOperand, 4> Ops; 4374 if (Op.getOpcode() == ISD::SHL_PARTS) { 4375 Ops.push_back(Tmp2); 4376 Ops.push_back(Tmp3); 4377 Ops.push_back(CC); 4378 Ops.push_back(Cond); 4379 Hi = DAG.getNode(X86ISD::CMOV, VT, &Ops[0], Ops.size()); 4380 4381 Ops.clear(); 4382 Ops.push_back(Tmp3); 4383 Ops.push_back(Tmp1); 4384 Ops.push_back(CC); 4385 Ops.push_back(Cond); 4386 Lo = DAG.getNode(X86ISD::CMOV, VT, &Ops[0], Ops.size()); 4387 } else { 4388 Ops.push_back(Tmp2); 4389 Ops.push_back(Tmp3); 4390 Ops.push_back(CC); 4391 Ops.push_back(Cond); 4392 Lo = DAG.getNode(X86ISD::CMOV, VT, &Ops[0], Ops.size()); 4393 4394 Ops.clear(); 4395 Ops.push_back(Tmp3); 4396 Ops.push_back(Tmp1); 4397 Ops.push_back(CC); 4398 Ops.push_back(Cond); 4399 Hi = DAG.getNode(X86ISD::CMOV, VT, &Ops[0], Ops.size()); 4400 } 4401 4402 VTs = DAG.getNodeValueTypes(VT, VT); 4403 Ops.clear(); 4404 Ops.push_back(Lo); 4405 Ops.push_back(Hi); 4406 return DAG.getNode(ISD::MERGE_VALUES, VTs, 2, &Ops[0], Ops.size()); 4407} 4408 4409SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { 4410 MVT SrcVT = Op.getOperand(0).getValueType(); 4411 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4412 "Unknown SINT_TO_FP to lower!"); 4413 4414 // These are really Legal; caller falls through into that case. 4415 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4416 return SDOperand(); 4417 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4418 Subtarget->is64Bit()) 4419 return SDOperand(); 4420 4421 unsigned Size = SrcVT.getSizeInBits()/8; 4422 MachineFunction &MF = DAG.getMachineFunction(); 4423 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4424 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4425 SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4426 StackSlot, 4427 PseudoSourceValue::getFixedStack(), 4428 SSFI); 4429 4430 // Build the FILD 4431 SDVTList Tys; 4432 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4433 if (useSSE) 4434 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4435 else 4436 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4437 SmallVector<SDOperand, 8> Ops; 4438 Ops.push_back(Chain); 4439 Ops.push_back(StackSlot); 4440 Ops.push_back(DAG.getValueType(SrcVT)); 4441 SDOperand Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4442 Tys, &Ops[0], Ops.size()); 4443 4444 if (useSSE) { 4445 Chain = Result.getValue(1); 4446 SDOperand InFlag = Result.getValue(2); 4447 4448 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4449 // shouldn't be necessary except that RFP cannot be live across 4450 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4451 MachineFunction &MF = DAG.getMachineFunction(); 4452 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4453 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4454 Tys = DAG.getVTList(MVT::Other); 4455 SmallVector<SDOperand, 8> Ops; 4456 Ops.push_back(Chain); 4457 Ops.push_back(Result); 4458 Ops.push_back(StackSlot); 4459 Ops.push_back(DAG.getValueType(Op.getValueType())); 4460 Ops.push_back(InFlag); 4461 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4462 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4463 PseudoSourceValue::getFixedStack(), SSFI); 4464 } 4465 4466 return Result; 4467} 4468 4469std::pair<SDOperand,SDOperand> X86TargetLowering:: 4470FP_TO_SINTHelper(SDOperand Op, SelectionDAG &DAG) { 4471 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4472 Op.getValueType().getSimpleVT() >= MVT::i16 && 4473 "Unknown FP_TO_SINT to lower!"); 4474 4475 // These are really Legal. 4476 if (Op.getValueType() == MVT::i32 && 4477 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4478 return std::make_pair(SDOperand(), SDOperand()); 4479 if (Subtarget->is64Bit() && 4480 Op.getValueType() == MVT::i64 && 4481 Op.getOperand(0).getValueType() != MVT::f80) 4482 return std::make_pair(SDOperand(), SDOperand()); 4483 4484 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4485 // stack slot. 4486 MachineFunction &MF = DAG.getMachineFunction(); 4487 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4488 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4489 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4490 unsigned Opc; 4491 switch (Op.getValueType().getSimpleVT()) { 4492 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4493 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4494 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4495 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4496 } 4497 4498 SDOperand Chain = DAG.getEntryNode(); 4499 SDOperand Value = Op.getOperand(0); 4500 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4501 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4502 Chain = DAG.getStore(Chain, Value, StackSlot, 4503 PseudoSourceValue::getFixedStack(), SSFI); 4504 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4505 SDOperand Ops[] = { 4506 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4507 }; 4508 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4509 Chain = Value.getValue(1); 4510 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4511 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4512 } 4513 4514 // Build the FP_TO_INT*_IN_MEM 4515 SDOperand Ops[] = { Chain, Value, StackSlot }; 4516 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4517 4518 return std::make_pair(FIST, StackSlot); 4519} 4520 4521SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { 4522 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(Op, DAG); 4523 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4524 if (FIST.Val == 0) return SDOperand(); 4525 4526 // Load the result. 4527 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4528} 4529 4530SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4531 std::pair<SDOperand,SDOperand> Vals = FP_TO_SINTHelper(SDOperand(N, 0), DAG); 4532 SDOperand FIST = Vals.first, StackSlot = Vals.second; 4533 if (FIST.Val == 0) return 0; 4534 4535 // Return an i64 load from the stack slot. 4536 SDOperand Res = DAG.getLoad(MVT::i64, FIST, StackSlot, NULL, 0); 4537 4538 // Use a MERGE_VALUES node to drop the chain result value. 4539 return DAG.getNode(ISD::MERGE_VALUES, MVT::i64, Res).Val; 4540} 4541 4542SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) { 4543 MVT VT = Op.getValueType(); 4544 MVT EltVT = VT; 4545 if (VT.isVector()) 4546 EltVT = VT.getVectorElementType(); 4547 std::vector<Constant*> CV; 4548 if (EltVT == MVT::f64) { 4549 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4550 CV.push_back(C); 4551 CV.push_back(C); 4552 } else { 4553 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4554 CV.push_back(C); 4555 CV.push_back(C); 4556 CV.push_back(C); 4557 CV.push_back(C); 4558 } 4559 Constant *C = ConstantVector::get(CV); 4560 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4561 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4562 PseudoSourceValue::getConstantPool(), 0, 4563 false, 16); 4564 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4565} 4566 4567SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) { 4568 MVT VT = Op.getValueType(); 4569 MVT EltVT = VT; 4570 unsigned EltNum = 1; 4571 if (VT.isVector()) { 4572 EltVT = VT.getVectorElementType(); 4573 EltNum = VT.getVectorNumElements(); 4574 } 4575 std::vector<Constant*> CV; 4576 if (EltVT == MVT::f64) { 4577 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4578 CV.push_back(C); 4579 CV.push_back(C); 4580 } else { 4581 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4582 CV.push_back(C); 4583 CV.push_back(C); 4584 CV.push_back(C); 4585 CV.push_back(C); 4586 } 4587 Constant *C = ConstantVector::get(CV); 4588 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4589 SDOperand Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4590 PseudoSourceValue::getConstantPool(), 0, 4591 false, 16); 4592 if (VT.isVector()) { 4593 return DAG.getNode(ISD::BIT_CONVERT, VT, 4594 DAG.getNode(ISD::XOR, MVT::v2i64, 4595 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4596 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4597 } else { 4598 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4599 } 4600} 4601 4602SDOperand X86TargetLowering::LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) { 4603 SDOperand Op0 = Op.getOperand(0); 4604 SDOperand Op1 = Op.getOperand(1); 4605 MVT VT = Op.getValueType(); 4606 MVT SrcVT = Op1.getValueType(); 4607 4608 // If second operand is smaller, extend it first. 4609 if (SrcVT.bitsLT(VT)) { 4610 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4611 SrcVT = VT; 4612 } 4613 // And if it is bigger, shrink it first. 4614 if (SrcVT.bitsGT(VT)) { 4615 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4616 SrcVT = VT; 4617 } 4618 4619 // At this point the operands and the result should have the same 4620 // type, and that won't be f80 since that is not custom lowered. 4621 4622 // First get the sign bit of second operand. 4623 std::vector<Constant*> CV; 4624 if (SrcVT == MVT::f64) { 4625 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4626 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4627 } else { 4628 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4629 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4630 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4631 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4632 } 4633 Constant *C = ConstantVector::get(CV); 4634 SDOperand CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4635 SDOperand Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4636 PseudoSourceValue::getConstantPool(), 0, 4637 false, 16); 4638 SDOperand SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4639 4640 // Shift sign bit right or left if the two operands have different types. 4641 if (SrcVT.bitsGT(VT)) { 4642 // Op0 is MVT::f32, Op1 is MVT::f64. 4643 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4644 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4645 DAG.getConstant(32, MVT::i32)); 4646 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4647 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4648 DAG.getIntPtrConstant(0)); 4649 } 4650 4651 // Clear first operand sign bit. 4652 CV.clear(); 4653 if (VT == MVT::f64) { 4654 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4655 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4656 } else { 4657 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4658 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4659 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4660 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4661 } 4662 C = ConstantVector::get(CV); 4663 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4664 SDOperand Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4665 PseudoSourceValue::getConstantPool(), 0, 4666 false, 16); 4667 SDOperand Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4668 4669 // Or the value with the sign bit. 4670 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4671} 4672 4673SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG) { 4674 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4675 SDOperand Cond; 4676 SDOperand Op0 = Op.getOperand(0); 4677 SDOperand Op1 = Op.getOperand(1); 4678 SDOperand CC = Op.getOperand(2); 4679 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4680 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4681 unsigned X86CC; 4682 4683 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4684 Op0, Op1, DAG)) { 4685 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4686 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4687 DAG.getConstant(X86CC, MVT::i8), Cond); 4688 } 4689 4690 assert(isFP && "Illegal integer SetCC!"); 4691 4692 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4693 switch (SetCCOpcode) { 4694 default: assert(false && "Illegal floating point SetCC!"); 4695 case ISD::SETOEQ: { // !PF & ZF 4696 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4697 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4698 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4699 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4700 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4701 } 4702 case ISD::SETUNE: { // PF | !ZF 4703 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4704 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4705 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4706 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4707 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4708 } 4709 } 4710} 4711 4712 4713SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) { 4714 bool addTest = true; 4715 SDOperand Cond = Op.getOperand(0); 4716 SDOperand CC; 4717 4718 if (Cond.getOpcode() == ISD::SETCC) 4719 Cond = LowerSETCC(Cond, DAG); 4720 4721 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4722 // setting operand in place of the X86ISD::SETCC. 4723 if (Cond.getOpcode() == X86ISD::SETCC) { 4724 CC = Cond.getOperand(0); 4725 4726 SDOperand Cmp = Cond.getOperand(1); 4727 unsigned Opc = Cmp.getOpcode(); 4728 MVT VT = Op.getValueType(); 4729 4730 bool IllegalFPCMov = false; 4731 if (VT.isFloatingPoint() && !VT.isVector() && 4732 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4733 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4734 4735 if ((Opc == X86ISD::CMP || 4736 Opc == X86ISD::COMI || 4737 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4738 Cond = Cmp; 4739 addTest = false; 4740 } 4741 } 4742 4743 if (addTest) { 4744 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4745 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4746 } 4747 4748 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4749 MVT::Flag); 4750 SmallVector<SDOperand, 4> Ops; 4751 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4752 // condition is true. 4753 Ops.push_back(Op.getOperand(2)); 4754 Ops.push_back(Op.getOperand(1)); 4755 Ops.push_back(CC); 4756 Ops.push_back(Cond); 4757 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4758} 4759 4760SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) { 4761 bool addTest = true; 4762 SDOperand Chain = Op.getOperand(0); 4763 SDOperand Cond = Op.getOperand(1); 4764 SDOperand Dest = Op.getOperand(2); 4765 SDOperand CC; 4766 4767 if (Cond.getOpcode() == ISD::SETCC) 4768 Cond = LowerSETCC(Cond, DAG); 4769 4770 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4771 // setting operand in place of the X86ISD::SETCC. 4772 if (Cond.getOpcode() == X86ISD::SETCC) { 4773 CC = Cond.getOperand(0); 4774 4775 SDOperand Cmp = Cond.getOperand(1); 4776 unsigned Opc = Cmp.getOpcode(); 4777 if (Opc == X86ISD::CMP || 4778 Opc == X86ISD::COMI || 4779 Opc == X86ISD::UCOMI) { 4780 Cond = Cmp; 4781 addTest = false; 4782 } 4783 } 4784 4785 if (addTest) { 4786 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4787 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4788 } 4789 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 4790 Chain, Op.getOperand(2), CC, Cond); 4791} 4792 4793 4794// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 4795// Calls to _alloca is needed to probe the stack when allocating more than 4k 4796// bytes in one go. Touching the stack at 4K increments is necessary to ensure 4797// that the guard pages used by the OS virtual memory manager are allocated in 4798// correct sequence. 4799SDOperand 4800X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op, 4801 SelectionDAG &DAG) { 4802 assert(Subtarget->isTargetCygMing() && 4803 "This should be used only on Cygwin/Mingw targets"); 4804 4805 // Get the inputs. 4806 SDOperand Chain = Op.getOperand(0); 4807 SDOperand Size = Op.getOperand(1); 4808 // FIXME: Ensure alignment here 4809 4810 SDOperand Flag; 4811 4812 MVT IntPtr = getPointerTy(); 4813 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 4814 4815 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 4816 4817 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 4818 Flag = Chain.getValue(1); 4819 4820 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4821 SDOperand Ops[] = { Chain, 4822 DAG.getTargetExternalSymbol("_alloca", IntPtr), 4823 DAG.getRegister(X86::EAX, IntPtr), 4824 DAG.getRegister(X86StackPtr, SPTy), 4825 Flag }; 4826 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 4827 Flag = Chain.getValue(1); 4828 4829 Chain = DAG.getCALLSEQ_END(Chain, 4830 DAG.getIntPtrConstant(0), 4831 DAG.getIntPtrConstant(0), 4832 Flag); 4833 4834 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 4835 4836 std::vector<MVT> Tys; 4837 Tys.push_back(SPTy); 4838 Tys.push_back(MVT::Other); 4839 SDOperand Ops1[2] = { Chain.getValue(0), Chain }; 4840 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops1, 2); 4841} 4842 4843SDOperand 4844X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 4845 SDOperand Chain, 4846 SDOperand Dst, SDOperand Src, 4847 SDOperand Size, unsigned Align, 4848 const Value *DstSV, uint64_t DstSVOff) { 4849 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4850 4851 /// If not DWORD aligned or size is more than the threshold, call the library. 4852 /// The libc version is likely to be faster for these cases. It can use the 4853 /// address value and run time information about the CPU. 4854 if ((Align & 3) == 0 || 4855 !ConstantSize || 4856 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 4857 SDOperand InFlag(0, 0); 4858 4859 // Check to see if there is a specialized entry-point for memory zeroing. 4860 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 4861 if (const char *bzeroEntry = 4862 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 4863 MVT IntPtr = getPointerTy(); 4864 const Type *IntPtrTy = getTargetData()->getIntPtrType(); 4865 TargetLowering::ArgListTy Args; 4866 TargetLowering::ArgListEntry Entry; 4867 Entry.Node = Dst; 4868 Entry.Ty = IntPtrTy; 4869 Args.push_back(Entry); 4870 Entry.Node = Size; 4871 Args.push_back(Entry); 4872 std::pair<SDOperand,SDOperand> CallResult = 4873 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 4874 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 4875 Args, DAG); 4876 return CallResult.second; 4877 } 4878 4879 // Otherwise have the target-independent code call memset. 4880 return SDOperand(); 4881 } 4882 4883 uint64_t SizeVal = ConstantSize->getValue(); 4884 SDOperand InFlag(0, 0); 4885 MVT AVT; 4886 SDOperand Count; 4887 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 4888 unsigned BytesLeft = 0; 4889 bool TwoRepStos = false; 4890 if (ValC) { 4891 unsigned ValReg; 4892 uint64_t Val = ValC->getValue() & 255; 4893 4894 // If the value is a constant, then we can potentially use larger sets. 4895 switch (Align & 3) { 4896 case 2: // WORD aligned 4897 AVT = MVT::i16; 4898 ValReg = X86::AX; 4899 Val = (Val << 8) | Val; 4900 break; 4901 case 0: // DWORD aligned 4902 AVT = MVT::i32; 4903 ValReg = X86::EAX; 4904 Val = (Val << 8) | Val; 4905 Val = (Val << 16) | Val; 4906 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 4907 AVT = MVT::i64; 4908 ValReg = X86::RAX; 4909 Val = (Val << 32) | Val; 4910 } 4911 break; 4912 default: // Byte aligned 4913 AVT = MVT::i8; 4914 ValReg = X86::AL; 4915 Count = DAG.getIntPtrConstant(SizeVal); 4916 break; 4917 } 4918 4919 if (AVT.bitsGT(MVT::i8)) { 4920 unsigned UBytes = AVT.getSizeInBits() / 8; 4921 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 4922 BytesLeft = SizeVal % UBytes; 4923 } 4924 4925 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 4926 InFlag); 4927 InFlag = Chain.getValue(1); 4928 } else { 4929 AVT = MVT::i8; 4930 Count = DAG.getIntPtrConstant(SizeVal); 4931 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 4932 InFlag = Chain.getValue(1); 4933 } 4934 4935 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 4936 Count, InFlag); 4937 InFlag = Chain.getValue(1); 4938 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 4939 Dst, InFlag); 4940 InFlag = Chain.getValue(1); 4941 4942 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 4943 SmallVector<SDOperand, 8> Ops; 4944 Ops.push_back(Chain); 4945 Ops.push_back(DAG.getValueType(AVT)); 4946 Ops.push_back(InFlag); 4947 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 4948 4949 if (TwoRepStos) { 4950 InFlag = Chain.getValue(1); 4951 Count = Size; 4952 MVT CVT = Count.getValueType(); 4953 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count, 4954 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 4955 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 4956 Left, InFlag); 4957 InFlag = Chain.getValue(1); 4958 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 4959 Ops.clear(); 4960 Ops.push_back(Chain); 4961 Ops.push_back(DAG.getValueType(MVT::i8)); 4962 Ops.push_back(InFlag); 4963 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 4964 } else if (BytesLeft) { 4965 // Handle the last 1 - 7 bytes. 4966 unsigned Offset = SizeVal - BytesLeft; 4967 MVT AddrVT = Dst.getValueType(); 4968 MVT SizeVT = Size.getValueType(); 4969 4970 Chain = DAG.getMemset(Chain, 4971 DAG.getNode(ISD::ADD, AddrVT, Dst, 4972 DAG.getConstant(Offset, AddrVT)), 4973 Src, 4974 DAG.getConstant(BytesLeft, SizeVT), 4975 Align, DstSV, DstSVOff + Offset); 4976 } 4977 4978 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 4979 return Chain; 4980} 4981 4982SDOperand 4983X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 4984 SDOperand Chain, 4985 SDOperand Dst, SDOperand Src, 4986 SDOperand Size, unsigned Align, 4987 bool AlwaysInline, 4988 const Value *DstSV, uint64_t DstSVOff, 4989 const Value *SrcSV, uint64_t SrcSVOff){ 4990 4991 // This requires the copy size to be a constant, preferrably 4992 // within a subtarget-specific limit. 4993 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4994 if (!ConstantSize) 4995 return SDOperand(); 4996 uint64_t SizeVal = ConstantSize->getValue(); 4997 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 4998 return SDOperand(); 4999 5000 MVT AVT; 5001 unsigned BytesLeft = 0; 5002 if (Align >= 8 && Subtarget->is64Bit()) 5003 AVT = MVT::i64; 5004 else if (Align >= 4) 5005 AVT = MVT::i32; 5006 else if (Align >= 2) 5007 AVT = MVT::i16; 5008 else 5009 AVT = MVT::i8; 5010 5011 unsigned UBytes = AVT.getSizeInBits() / 8; 5012 unsigned CountVal = SizeVal / UBytes; 5013 SDOperand Count = DAG.getIntPtrConstant(CountVal); 5014 BytesLeft = SizeVal % UBytes; 5015 5016 SDOperand InFlag(0, 0); 5017 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5018 Count, InFlag); 5019 InFlag = Chain.getValue(1); 5020 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5021 Dst, InFlag); 5022 InFlag = Chain.getValue(1); 5023 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 5024 Src, InFlag); 5025 InFlag = Chain.getValue(1); 5026 5027 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5028 SmallVector<SDOperand, 8> Ops; 5029 Ops.push_back(Chain); 5030 Ops.push_back(DAG.getValueType(AVT)); 5031 Ops.push_back(InFlag); 5032 SDOperand RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5033 5034 SmallVector<SDOperand, 4> Results; 5035 Results.push_back(RepMovs); 5036 if (BytesLeft) { 5037 // Handle the last 1 - 7 bytes. 5038 unsigned Offset = SizeVal - BytesLeft; 5039 MVT DstVT = Dst.getValueType(); 5040 MVT SrcVT = Src.getValueType(); 5041 MVT SizeVT = Size.getValueType(); 5042 Results.push_back(DAG.getMemcpy(Chain, 5043 DAG.getNode(ISD::ADD, DstVT, Dst, 5044 DAG.getConstant(Offset, DstVT)), 5045 DAG.getNode(ISD::ADD, SrcVT, Src, 5046 DAG.getConstant(Offset, SrcVT)), 5047 DAG.getConstant(BytesLeft, SizeVT), 5048 Align, AlwaysInline, 5049 DstSV, DstSVOff + Offset, 5050 SrcSV, SrcSVOff + Offset)); 5051 } 5052 5053 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5054} 5055 5056/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5057SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5058 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5059 SDOperand TheChain = N->getOperand(0); 5060 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5061 if (Subtarget->is64Bit()) { 5062 SDOperand rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5063 SDOperand rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5064 MVT::i64, rax.getValue(2)); 5065 SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5066 DAG.getConstant(32, MVT::i8)); 5067 SDOperand Ops[] = { 5068 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5069 }; 5070 5071 Tys = DAG.getVTList(MVT::i64, MVT::Other); 5072 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2).Val; 5073 } 5074 5075 SDOperand eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5076 SDOperand edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5077 MVT::i32, eax.getValue(2)); 5078 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5079 SDOperand Ops[] = { eax, edx }; 5080 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5081 5082 // Use a MERGE_VALUES to return the value and chain. 5083 Ops[1] = edx.getValue(1); 5084 Tys = DAG.getVTList(MVT::i64, MVT::Other); 5085 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2).Val; 5086} 5087 5088SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) { 5089 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5090 5091 if (!Subtarget->is64Bit()) { 5092 // vastart just stores the address of the VarArgsFrameIndex slot into the 5093 // memory location argument. 5094 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5095 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5096 } 5097 5098 // __va_list_tag: 5099 // gp_offset (0 - 6 * 8) 5100 // fp_offset (48 - 48 + 8 * 16) 5101 // overflow_arg_area (point to parameters coming in memory). 5102 // reg_save_area 5103 SmallVector<SDOperand, 8> MemOps; 5104 SDOperand FIN = Op.getOperand(1); 5105 // Store gp_offset 5106 SDOperand Store = DAG.getStore(Op.getOperand(0), 5107 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5108 FIN, SV, 0); 5109 MemOps.push_back(Store); 5110 5111 // Store fp_offset 5112 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5113 Store = DAG.getStore(Op.getOperand(0), 5114 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5115 FIN, SV, 0); 5116 MemOps.push_back(Store); 5117 5118 // Store ptr to overflow_arg_area 5119 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5120 SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5121 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5122 MemOps.push_back(Store); 5123 5124 // Store ptr to reg_save_area. 5125 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5126 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5127 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5128 MemOps.push_back(Store); 5129 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5130} 5131 5132SDOperand X86TargetLowering::LowerVAARG(SDOperand Op, SelectionDAG &DAG) { 5133 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5134 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5135 SDOperand Chain = Op.getOperand(0); 5136 SDOperand SrcPtr = Op.getOperand(1); 5137 SDOperand SrcSV = Op.getOperand(2); 5138 5139 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5140 abort(); 5141 return SDOperand(); 5142} 5143 5144SDOperand X86TargetLowering::LowerVACOPY(SDOperand Op, SelectionDAG &DAG) { 5145 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5146 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5147 SDOperand Chain = Op.getOperand(0); 5148 SDOperand DstPtr = Op.getOperand(1); 5149 SDOperand SrcPtr = Op.getOperand(2); 5150 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5151 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5152 5153 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5154 DAG.getIntPtrConstant(24), 8, false, 5155 DstSV, 0, SrcSV, 0); 5156} 5157 5158SDOperand 5159X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) { 5160 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5161 switch (IntNo) { 5162 default: return SDOperand(); // Don't custom lower most intrinsics. 5163 // Comparison intrinsics. 5164 case Intrinsic::x86_sse_comieq_ss: 5165 case Intrinsic::x86_sse_comilt_ss: 5166 case Intrinsic::x86_sse_comile_ss: 5167 case Intrinsic::x86_sse_comigt_ss: 5168 case Intrinsic::x86_sse_comige_ss: 5169 case Intrinsic::x86_sse_comineq_ss: 5170 case Intrinsic::x86_sse_ucomieq_ss: 5171 case Intrinsic::x86_sse_ucomilt_ss: 5172 case Intrinsic::x86_sse_ucomile_ss: 5173 case Intrinsic::x86_sse_ucomigt_ss: 5174 case Intrinsic::x86_sse_ucomige_ss: 5175 case Intrinsic::x86_sse_ucomineq_ss: 5176 case Intrinsic::x86_sse2_comieq_sd: 5177 case Intrinsic::x86_sse2_comilt_sd: 5178 case Intrinsic::x86_sse2_comile_sd: 5179 case Intrinsic::x86_sse2_comigt_sd: 5180 case Intrinsic::x86_sse2_comige_sd: 5181 case Intrinsic::x86_sse2_comineq_sd: 5182 case Intrinsic::x86_sse2_ucomieq_sd: 5183 case Intrinsic::x86_sse2_ucomilt_sd: 5184 case Intrinsic::x86_sse2_ucomile_sd: 5185 case Intrinsic::x86_sse2_ucomigt_sd: 5186 case Intrinsic::x86_sse2_ucomige_sd: 5187 case Intrinsic::x86_sse2_ucomineq_sd: { 5188 unsigned Opc = 0; 5189 ISD::CondCode CC = ISD::SETCC_INVALID; 5190 switch (IntNo) { 5191 default: break; 5192 case Intrinsic::x86_sse_comieq_ss: 5193 case Intrinsic::x86_sse2_comieq_sd: 5194 Opc = X86ISD::COMI; 5195 CC = ISD::SETEQ; 5196 break; 5197 case Intrinsic::x86_sse_comilt_ss: 5198 case Intrinsic::x86_sse2_comilt_sd: 5199 Opc = X86ISD::COMI; 5200 CC = ISD::SETLT; 5201 break; 5202 case Intrinsic::x86_sse_comile_ss: 5203 case Intrinsic::x86_sse2_comile_sd: 5204 Opc = X86ISD::COMI; 5205 CC = ISD::SETLE; 5206 break; 5207 case Intrinsic::x86_sse_comigt_ss: 5208 case Intrinsic::x86_sse2_comigt_sd: 5209 Opc = X86ISD::COMI; 5210 CC = ISD::SETGT; 5211 break; 5212 case Intrinsic::x86_sse_comige_ss: 5213 case Intrinsic::x86_sse2_comige_sd: 5214 Opc = X86ISD::COMI; 5215 CC = ISD::SETGE; 5216 break; 5217 case Intrinsic::x86_sse_comineq_ss: 5218 case Intrinsic::x86_sse2_comineq_sd: 5219 Opc = X86ISD::COMI; 5220 CC = ISD::SETNE; 5221 break; 5222 case Intrinsic::x86_sse_ucomieq_ss: 5223 case Intrinsic::x86_sse2_ucomieq_sd: 5224 Opc = X86ISD::UCOMI; 5225 CC = ISD::SETEQ; 5226 break; 5227 case Intrinsic::x86_sse_ucomilt_ss: 5228 case Intrinsic::x86_sse2_ucomilt_sd: 5229 Opc = X86ISD::UCOMI; 5230 CC = ISD::SETLT; 5231 break; 5232 case Intrinsic::x86_sse_ucomile_ss: 5233 case Intrinsic::x86_sse2_ucomile_sd: 5234 Opc = X86ISD::UCOMI; 5235 CC = ISD::SETLE; 5236 break; 5237 case Intrinsic::x86_sse_ucomigt_ss: 5238 case Intrinsic::x86_sse2_ucomigt_sd: 5239 Opc = X86ISD::UCOMI; 5240 CC = ISD::SETGT; 5241 break; 5242 case Intrinsic::x86_sse_ucomige_ss: 5243 case Intrinsic::x86_sse2_ucomige_sd: 5244 Opc = X86ISD::UCOMI; 5245 CC = ISD::SETGE; 5246 break; 5247 case Intrinsic::x86_sse_ucomineq_ss: 5248 case Intrinsic::x86_sse2_ucomineq_sd: 5249 Opc = X86ISD::UCOMI; 5250 CC = ISD::SETNE; 5251 break; 5252 } 5253 5254 unsigned X86CC; 5255 SDOperand LHS = Op.getOperand(1); 5256 SDOperand RHS = Op.getOperand(2); 5257 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5258 5259 SDOperand Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5260 SDOperand SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5261 DAG.getConstant(X86CC, MVT::i8), Cond); 5262 return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC); 5263 } 5264 5265 // Fix vector shift instructions where the last operand is a non-immediate 5266 // i32 value. 5267 case Intrinsic::x86_sse2_pslli_w: 5268 case Intrinsic::x86_sse2_pslli_d: 5269 case Intrinsic::x86_sse2_pslli_q: 5270 case Intrinsic::x86_sse2_psrli_w: 5271 case Intrinsic::x86_sse2_psrli_d: 5272 case Intrinsic::x86_sse2_psrli_q: 5273 case Intrinsic::x86_sse2_psrai_w: 5274 case Intrinsic::x86_sse2_psrai_d: 5275 case Intrinsic::x86_mmx_pslli_w: 5276 case Intrinsic::x86_mmx_pslli_d: 5277 case Intrinsic::x86_mmx_pslli_q: 5278 case Intrinsic::x86_mmx_psrli_w: 5279 case Intrinsic::x86_mmx_psrli_d: 5280 case Intrinsic::x86_mmx_psrli_q: 5281 case Intrinsic::x86_mmx_psrai_w: 5282 case Intrinsic::x86_mmx_psrai_d: { 5283 SDOperand ShAmt = Op.getOperand(2); 5284 if (isa<ConstantSDNode>(ShAmt)) 5285 return SDOperand(); 5286 5287 unsigned NewIntNo = 0; 5288 MVT ShAmtVT = MVT::v4i32; 5289 switch (IntNo) { 5290 case Intrinsic::x86_sse2_pslli_w: 5291 NewIntNo = Intrinsic::x86_sse2_psll_w; 5292 break; 5293 case Intrinsic::x86_sse2_pslli_d: 5294 NewIntNo = Intrinsic::x86_sse2_psll_d; 5295 break; 5296 case Intrinsic::x86_sse2_pslli_q: 5297 NewIntNo = Intrinsic::x86_sse2_psll_q; 5298 break; 5299 case Intrinsic::x86_sse2_psrli_w: 5300 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5301 break; 5302 case Intrinsic::x86_sse2_psrli_d: 5303 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5304 break; 5305 case Intrinsic::x86_sse2_psrli_q: 5306 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5307 break; 5308 case Intrinsic::x86_sse2_psrai_w: 5309 NewIntNo = Intrinsic::x86_sse2_psra_w; 5310 break; 5311 case Intrinsic::x86_sse2_psrai_d: 5312 NewIntNo = Intrinsic::x86_sse2_psra_d; 5313 break; 5314 default: { 5315 ShAmtVT = MVT::v2i32; 5316 switch (IntNo) { 5317 case Intrinsic::x86_mmx_pslli_w: 5318 NewIntNo = Intrinsic::x86_mmx_psll_w; 5319 break; 5320 case Intrinsic::x86_mmx_pslli_d: 5321 NewIntNo = Intrinsic::x86_mmx_psll_d; 5322 break; 5323 case Intrinsic::x86_mmx_pslli_q: 5324 NewIntNo = Intrinsic::x86_mmx_psll_q; 5325 break; 5326 case Intrinsic::x86_mmx_psrli_w: 5327 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5328 break; 5329 case Intrinsic::x86_mmx_psrli_d: 5330 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5331 break; 5332 case Intrinsic::x86_mmx_psrli_q: 5333 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5334 break; 5335 case Intrinsic::x86_mmx_psrai_w: 5336 NewIntNo = Intrinsic::x86_mmx_psra_w; 5337 break; 5338 case Intrinsic::x86_mmx_psrai_d: 5339 NewIntNo = Intrinsic::x86_mmx_psra_d; 5340 break; 5341 default: abort(); // Can't reach here. 5342 } 5343 break; 5344 } 5345 } 5346 MVT VT = Op.getValueType(); 5347 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5348 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5349 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5350 DAG.getConstant(NewIntNo, MVT::i32), 5351 Op.getOperand(1), ShAmt); 5352 } 5353 } 5354} 5355 5356SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) { 5357 // Depths > 0 not supported yet! 5358 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5359 return SDOperand(); 5360 5361 // Just load the return address 5362 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5363 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5364} 5365 5366SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) { 5367 // Depths > 0 not supported yet! 5368 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5369 return SDOperand(); 5370 5371 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 5372 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5373 DAG.getIntPtrConstant(4)); 5374} 5375 5376SDOperand X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDOperand Op, 5377 SelectionDAG &DAG) { 5378 // Is not yet supported on x86-64 5379 if (Subtarget->is64Bit()) 5380 return SDOperand(); 5381 5382 return DAG.getIntPtrConstant(8); 5383} 5384 5385SDOperand X86TargetLowering::LowerEH_RETURN(SDOperand Op, SelectionDAG &DAG) 5386{ 5387 assert(!Subtarget->is64Bit() && 5388 "Lowering of eh_return builtin is not supported yet on x86-64"); 5389 5390 MachineFunction &MF = DAG.getMachineFunction(); 5391 SDOperand Chain = Op.getOperand(0); 5392 SDOperand Offset = Op.getOperand(1); 5393 SDOperand Handler = Op.getOperand(2); 5394 5395 SDOperand Frame = DAG.getRegister(RegInfo->getFrameRegister(MF), 5396 getPointerTy()); 5397 5398 SDOperand StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5399 DAG.getIntPtrConstant(-4UL)); 5400 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5401 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5402 Chain = DAG.getCopyToReg(Chain, X86::ECX, StoreAddr); 5403 MF.getRegInfo().addLiveOut(X86::ECX); 5404 5405 return DAG.getNode(X86ISD::EH_RETURN, MVT::Other, 5406 Chain, DAG.getRegister(X86::ECX, getPointerTy())); 5407} 5408 5409SDOperand X86TargetLowering::LowerTRAMPOLINE(SDOperand Op, 5410 SelectionDAG &DAG) { 5411 SDOperand Root = Op.getOperand(0); 5412 SDOperand Trmp = Op.getOperand(1); // trampoline 5413 SDOperand FPtr = Op.getOperand(2); // nested function 5414 SDOperand Nest = Op.getOperand(3); // 'nest' parameter value 5415 5416 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5417 5418 const X86InstrInfo *TII = 5419 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5420 5421 if (Subtarget->is64Bit()) { 5422 SDOperand OutChains[6]; 5423 5424 // Large code-model. 5425 5426 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5427 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5428 5429 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5430 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5431 5432 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5433 5434 // Load the pointer to the nested function into R11. 5435 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5436 SDOperand Addr = Trmp; 5437 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5438 TrmpAddr, 0); 5439 5440 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5441 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5442 5443 // Load the 'nest' parameter value into R10. 5444 // R10 is specified in X86CallingConv.td 5445 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5446 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5447 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5448 TrmpAddr, 10); 5449 5450 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5451 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5452 5453 // Jump to the nested function. 5454 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5455 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5456 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5457 TrmpAddr, 20); 5458 5459 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5460 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5461 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5462 TrmpAddr, 22); 5463 5464 SDOperand Ops[] = 5465 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5466 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(), Ops, 2); 5467 } else { 5468 const Function *Func = 5469 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5470 unsigned CC = Func->getCallingConv(); 5471 unsigned NestReg; 5472 5473 switch (CC) { 5474 default: 5475 assert(0 && "Unsupported calling convention"); 5476 case CallingConv::C: 5477 case CallingConv::X86_StdCall: { 5478 // Pass 'nest' parameter in ECX. 5479 // Must be kept in sync with X86CallingConv.td 5480 NestReg = X86::ECX; 5481 5482 // Check that ECX wasn't needed by an 'inreg' parameter. 5483 const FunctionType *FTy = Func->getFunctionType(); 5484 const PAListPtr &Attrs = Func->getParamAttrs(); 5485 5486 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5487 unsigned InRegCount = 0; 5488 unsigned Idx = 1; 5489 5490 for (FunctionType::param_iterator I = FTy->param_begin(), 5491 E = FTy->param_end(); I != E; ++I, ++Idx) 5492 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5493 // FIXME: should only count parameters that are lowered to integers. 5494 InRegCount += (getTargetData()->getTypeSizeInBits(*I) + 31) / 32; 5495 5496 if (InRegCount > 2) { 5497 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5498 abort(); 5499 } 5500 } 5501 break; 5502 } 5503 case CallingConv::X86_FastCall: 5504 // Pass 'nest' parameter in EAX. 5505 // Must be kept in sync with X86CallingConv.td 5506 NestReg = X86::EAX; 5507 break; 5508 } 5509 5510 SDOperand OutChains[4]; 5511 SDOperand Addr, Disp; 5512 5513 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5514 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5515 5516 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5517 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5518 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5519 Trmp, TrmpAddr, 0); 5520 5521 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5522 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5523 5524 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5525 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5526 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5527 TrmpAddr, 5, false, 1); 5528 5529 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5530 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5531 5532 SDOperand Ops[] = 5533 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5534 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(), Ops, 2); 5535 } 5536} 5537 5538SDOperand X86TargetLowering::LowerFLT_ROUNDS_(SDOperand Op, SelectionDAG &DAG) { 5539 /* 5540 The rounding mode is in bits 11:10 of FPSR, and has the following 5541 settings: 5542 00 Round to nearest 5543 01 Round to -inf 5544 10 Round to +inf 5545 11 Round to 0 5546 5547 FLT_ROUNDS, on the other hand, expects the following: 5548 -1 Undefined 5549 0 Round to 0 5550 1 Round to nearest 5551 2 Round to +inf 5552 3 Round to -inf 5553 5554 To perform the conversion, we do: 5555 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5556 */ 5557 5558 MachineFunction &MF = DAG.getMachineFunction(); 5559 const TargetMachine &TM = MF.getTarget(); 5560 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5561 unsigned StackAlignment = TFI.getStackAlignment(); 5562 MVT VT = Op.getValueType(); 5563 5564 // Save FP Control Word to stack slot 5565 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5566 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5567 5568 SDOperand Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5569 DAG.getEntryNode(), StackSlot); 5570 5571 // Load FP Control Word from stack slot 5572 SDOperand CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5573 5574 // Transform as necessary 5575 SDOperand CWD1 = 5576 DAG.getNode(ISD::SRL, MVT::i16, 5577 DAG.getNode(ISD::AND, MVT::i16, 5578 CWD, DAG.getConstant(0x800, MVT::i16)), 5579 DAG.getConstant(11, MVT::i8)); 5580 SDOperand CWD2 = 5581 DAG.getNode(ISD::SRL, MVT::i16, 5582 DAG.getNode(ISD::AND, MVT::i16, 5583 CWD, DAG.getConstant(0x400, MVT::i16)), 5584 DAG.getConstant(9, MVT::i8)); 5585 5586 SDOperand RetVal = 5587 DAG.getNode(ISD::AND, MVT::i16, 5588 DAG.getNode(ISD::ADD, MVT::i16, 5589 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5590 DAG.getConstant(1, MVT::i16)), 5591 DAG.getConstant(3, MVT::i16)); 5592 5593 5594 return DAG.getNode((VT.getSizeInBits() < 16 ? 5595 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5596} 5597 5598SDOperand X86TargetLowering::LowerCTLZ(SDOperand Op, SelectionDAG &DAG) { 5599 MVT VT = Op.getValueType(); 5600 MVT OpVT = VT; 5601 unsigned NumBits = VT.getSizeInBits(); 5602 5603 Op = Op.getOperand(0); 5604 if (VT == MVT::i8) { 5605 // Zero extend to i32 since there is not an i8 bsr. 5606 OpVT = MVT::i32; 5607 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5608 } 5609 5610 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5611 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5612 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5613 5614 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5615 SmallVector<SDOperand, 4> Ops; 5616 Ops.push_back(Op); 5617 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5618 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5619 Ops.push_back(Op.getValue(1)); 5620 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5621 5622 // Finally xor with NumBits-1. 5623 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5624 5625 if (VT == MVT::i8) 5626 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5627 return Op; 5628} 5629 5630SDOperand X86TargetLowering::LowerCTTZ(SDOperand Op, SelectionDAG &DAG) { 5631 MVT VT = Op.getValueType(); 5632 MVT OpVT = VT; 5633 unsigned NumBits = VT.getSizeInBits(); 5634 5635 Op = Op.getOperand(0); 5636 if (VT == MVT::i8) { 5637 OpVT = MVT::i32; 5638 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5639 } 5640 5641 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5642 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5643 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5644 5645 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5646 SmallVector<SDOperand, 4> Ops; 5647 Ops.push_back(Op); 5648 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5649 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5650 Ops.push_back(Op.getValue(1)); 5651 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5652 5653 if (VT == MVT::i8) 5654 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5655 return Op; 5656} 5657 5658SDOperand X86TargetLowering::LowerCMP_SWAP(SDOperand Op, SelectionDAG &DAG) { 5659 MVT T = cast<AtomicSDNode>(Op.Val)->getVT(); 5660 unsigned Reg = 0; 5661 unsigned size = 0; 5662 switch(T.getSimpleVT()) { 5663 default: 5664 assert(false && "Invalid value type!"); 5665 case MVT::i8: Reg = X86::AL; size = 1; break; 5666 case MVT::i16: Reg = X86::AX; size = 2; break; 5667 case MVT::i32: Reg = X86::EAX; size = 4; break; 5668 case MVT::i64: 5669 if (Subtarget->is64Bit()) { 5670 Reg = X86::RAX; size = 8; 5671 } else //Should go away when LowerType stuff lands 5672 return SDOperand(ExpandATOMIC_CMP_SWAP(Op.Val, DAG), 0); 5673 break; 5674 }; 5675 SDOperand cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5676 Op.getOperand(3), SDOperand()); 5677 SDOperand Ops[] = { cpIn.getValue(0), 5678 Op.getOperand(1), 5679 Op.getOperand(2), 5680 DAG.getTargetConstant(size, MVT::i8), 5681 cpIn.getValue(1) }; 5682 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5683 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5684 SDOperand cpOut = 5685 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5686 return cpOut; 5687} 5688 5689SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, SelectionDAG &DAG) { 5690 MVT T = cast<AtomicSDNode>(Op)->getVT(); 5691 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5692 SDOperand cpInL, cpInH; 5693 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5694 DAG.getConstant(0, MVT::i32)); 5695 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5696 DAG.getConstant(1, MVT::i32)); 5697 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5698 cpInL, SDOperand()); 5699 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5700 cpInH, cpInL.getValue(1)); 5701 SDOperand swapInL, swapInH; 5702 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5703 DAG.getConstant(0, MVT::i32)); 5704 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5705 DAG.getConstant(1, MVT::i32)); 5706 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5707 swapInL, cpInH.getValue(1)); 5708 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5709 swapInH, swapInL.getValue(1)); 5710 SDOperand Ops[] = { swapInH.getValue(0), 5711 Op->getOperand(1), 5712 swapInH.getValue(1)}; 5713 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5714 SDOperand Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5715 SDOperand cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5716 Result.getValue(1)); 5717 SDOperand cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5718 cpOutL.getValue(2)); 5719 SDOperand OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5720 SDOperand ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5721 Tys = DAG.getVTList(MVT::i64, MVT::Other); 5722 return DAG.getNode(ISD::MERGE_VALUES, Tys, ResultVal, cpOutH.getValue(1)).Val; 5723} 5724 5725SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, SelectionDAG &DAG) { 5726 MVT T = cast<AtomicSDNode>(Op)->getVT(); 5727 assert (T == MVT::i32 && "Only know how to expand i32 Atomic Load Sub"); 5728 SDOperand negOp = DAG.getNode(ISD::SUB, T, 5729 DAG.getConstant(0, T), Op->getOperand(2)); 5730 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, Op->getOperand(0), 5731 Op->getOperand(1), negOp, T, 5732 cast<AtomicSDNode>(Op)->getSrcValue(), 5733 cast<AtomicSDNode>(Op)->getAlignment()).Val; 5734} 5735 5736/// LowerOperation - Provide custom lowering hooks for some operations. 5737/// 5738SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 5739 switch (Op.getOpcode()) { 5740 default: assert(0 && "Should not custom lower this!"); 5741 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 5742 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5743 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5744 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5745 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5746 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5747 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5748 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5749 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5750 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5751 case ISD::SHL_PARTS: 5752 case ISD::SRA_PARTS: 5753 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5754 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5755 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5756 case ISD::FABS: return LowerFABS(Op, DAG); 5757 case ISD::FNEG: return LowerFNEG(Op, DAG); 5758 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5759 case ISD::SETCC: return LowerSETCC(Op, DAG); 5760 case ISD::SELECT: return LowerSELECT(Op, DAG); 5761 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5762 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5763 case ISD::CALL: return LowerCALL(Op, DAG); 5764 case ISD::RET: return LowerRET(Op, DAG); 5765 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 5766 case ISD::VASTART: return LowerVASTART(Op, DAG); 5767 case ISD::VAARG: return LowerVAARG(Op, DAG); 5768 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 5769 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 5770 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 5771 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 5772 case ISD::FRAME_TO_ARGS_OFFSET: 5773 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 5774 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 5775 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 5776 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 5777 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 5778 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 5779 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 5780 5781 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 5782 case ISD::READCYCLECOUNTER: 5783 return SDOperand(ExpandREADCYCLECOUNTER(Op.Val, DAG), 0); 5784 } 5785} 5786 5787/// ExpandOperation - Provide custom lowering hooks for expanding operations. 5788SDNode *X86TargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) { 5789 switch (N->getOpcode()) { 5790 default: assert(0 && "Should not custom lower this!"); 5791 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 5792 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 5793 case ISD::ATOMIC_CMP_SWAP: return ExpandATOMIC_CMP_SWAP(N, DAG); 5794 case ISD::ATOMIC_LOAD_SUB: return ExpandATOMIC_LOAD_SUB(N,DAG); 5795 } 5796} 5797 5798const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 5799 switch (Opcode) { 5800 default: return NULL; 5801 case X86ISD::BSF: return "X86ISD::BSF"; 5802 case X86ISD::BSR: return "X86ISD::BSR"; 5803 case X86ISD::SHLD: return "X86ISD::SHLD"; 5804 case X86ISD::SHRD: return "X86ISD::SHRD"; 5805 case X86ISD::FAND: return "X86ISD::FAND"; 5806 case X86ISD::FOR: return "X86ISD::FOR"; 5807 case X86ISD::FXOR: return "X86ISD::FXOR"; 5808 case X86ISD::FSRL: return "X86ISD::FSRL"; 5809 case X86ISD::FILD: return "X86ISD::FILD"; 5810 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 5811 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 5812 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 5813 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 5814 case X86ISD::FLD: return "X86ISD::FLD"; 5815 case X86ISD::FST: return "X86ISD::FST"; 5816 case X86ISD::CALL: return "X86ISD::CALL"; 5817 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 5818 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 5819 case X86ISD::CMP: return "X86ISD::CMP"; 5820 case X86ISD::COMI: return "X86ISD::COMI"; 5821 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 5822 case X86ISD::SETCC: return "X86ISD::SETCC"; 5823 case X86ISD::CMOV: return "X86ISD::CMOV"; 5824 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 5825 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 5826 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 5827 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 5828 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 5829 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 5830 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 5831 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 5832 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 5833 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 5834 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 5835 case X86ISD::FMAX: return "X86ISD::FMAX"; 5836 case X86ISD::FMIN: return "X86ISD::FMIN"; 5837 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 5838 case X86ISD::FRCP: return "X86ISD::FRCP"; 5839 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 5840 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 5841 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 5842 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 5843 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 5844 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 5845 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 5846 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 5847 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 5848 case X86ISD::VSHL: return "X86ISD::VSHL"; 5849 case X86ISD::VSRL: return "X86ISD::VSRL"; 5850 } 5851} 5852 5853// isLegalAddressingMode - Return true if the addressing mode represented 5854// by AM is legal for this target, for a load/store of the specified type. 5855bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 5856 const Type *Ty) const { 5857 // X86 supports extremely general addressing modes. 5858 5859 // X86 allows a sign-extended 32-bit immediate field as a displacement. 5860 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 5861 return false; 5862 5863 if (AM.BaseGV) { 5864 // We can only fold this if we don't need an extra load. 5865 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 5866 return false; 5867 5868 // X86-64 only supports addr of globals in small code model. 5869 if (Subtarget->is64Bit()) { 5870 if (getTargetMachine().getCodeModel() != CodeModel::Small) 5871 return false; 5872 // If lower 4G is not available, then we must use rip-relative addressing. 5873 if (AM.BaseOffs || AM.Scale > 1) 5874 return false; 5875 } 5876 } 5877 5878 switch (AM.Scale) { 5879 case 0: 5880 case 1: 5881 case 2: 5882 case 4: 5883 case 8: 5884 // These scales always work. 5885 break; 5886 case 3: 5887 case 5: 5888 case 9: 5889 // These scales are formed with basereg+scalereg. Only accept if there is 5890 // no basereg yet. 5891 if (AM.HasBaseReg) 5892 return false; 5893 break; 5894 default: // Other stuff never works. 5895 return false; 5896 } 5897 5898 return true; 5899} 5900 5901 5902bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 5903 if (!Ty1->isInteger() || !Ty2->isInteger()) 5904 return false; 5905 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 5906 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 5907 if (NumBits1 <= NumBits2) 5908 return false; 5909 return Subtarget->is64Bit() || NumBits1 < 64; 5910} 5911 5912bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 5913 if (!VT1.isInteger() || !VT2.isInteger()) 5914 return false; 5915 unsigned NumBits1 = VT1.getSizeInBits(); 5916 unsigned NumBits2 = VT2.getSizeInBits(); 5917 if (NumBits1 <= NumBits2) 5918 return false; 5919 return Subtarget->is64Bit() || NumBits1 < 64; 5920} 5921 5922/// isShuffleMaskLegal - Targets can use this to indicate that they only 5923/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 5924/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 5925/// are assumed to be legal. 5926bool 5927X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT VT) const { 5928 // Only do shuffles on 128-bit vector types for now. 5929 if (VT.getSizeInBits() == 64) return false; 5930 return (Mask.Val->getNumOperands() <= 4 || 5931 isIdentityMask(Mask.Val) || 5932 isIdentityMask(Mask.Val, true) || 5933 isSplatMask(Mask.Val) || 5934 isPSHUFHW_PSHUFLWMask(Mask.Val) || 5935 X86::isUNPCKLMask(Mask.Val) || 5936 X86::isUNPCKHMask(Mask.Val) || 5937 X86::isUNPCKL_v_undef_Mask(Mask.Val) || 5938 X86::isUNPCKH_v_undef_Mask(Mask.Val)); 5939} 5940 5941bool 5942X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDOperand> &BVOps, 5943 MVT EVT, SelectionDAG &DAG) const { 5944 unsigned NumElts = BVOps.size(); 5945 // Only do shuffles on 128-bit vector types for now. 5946 if (EVT.getSizeInBits() * NumElts == 64) return false; 5947 if (NumElts == 2) return true; 5948 if (NumElts == 4) { 5949 return (isMOVLMask(&BVOps[0], 4) || 5950 isCommutedMOVL(&BVOps[0], 4, true) || 5951 isSHUFPMask(&BVOps[0], 4) || 5952 isCommutedSHUFP(&BVOps[0], 4)); 5953 } 5954 return false; 5955} 5956 5957//===----------------------------------------------------------------------===// 5958// X86 Scheduler Hooks 5959//===----------------------------------------------------------------------===// 5960 5961// private utility function 5962MachineBasicBlock * 5963X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 5964 MachineBasicBlock *MBB, 5965 unsigned regOpc, 5966 unsigned immOpc, 5967 bool invSrc) { 5968 // For the atomic bitwise operator, we generate 5969 // thisMBB: 5970 // newMBB: 5971 // ld t1 = [bitinstr.addr] 5972 // op t2 = t1, [bitinstr.val] 5973 // mov EAX = t1 5974 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 5975 // bz newMBB 5976 // fallthrough -->nextMBB 5977 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 5978 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 5979 ilist<MachineBasicBlock>::iterator MBBIter = MBB; 5980 ++MBBIter; 5981 5982 /// First build the CFG 5983 MachineFunction *F = MBB->getParent(); 5984 MachineBasicBlock *thisMBB = MBB; 5985 MachineBasicBlock *newMBB = new MachineBasicBlock(LLVM_BB); 5986 MachineBasicBlock *nextMBB = new MachineBasicBlock(LLVM_BB); 5987 F->getBasicBlockList().insert(MBBIter, newMBB); 5988 F->getBasicBlockList().insert(MBBIter, nextMBB); 5989 5990 // Move all successors to thisMBB to nextMBB 5991 nextMBB->transferSuccessors(thisMBB); 5992 5993 // Update thisMBB to fall through to newMBB 5994 thisMBB->addSuccessor(newMBB); 5995 5996 // newMBB jumps to itself and fall through to nextMBB 5997 newMBB->addSuccessor(nextMBB); 5998 newMBB->addSuccessor(newMBB); 5999 6000 // Insert instructions into newMBB based on incoming instruction 6001 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 6002 MachineOperand& destOper = bInstr->getOperand(0); 6003 MachineOperand* argOpers[6]; 6004 int numArgs = bInstr->getNumOperands() - 1; 6005 for (int i=0; i < numArgs; ++i) 6006 argOpers[i] = &bInstr->getOperand(i+1); 6007 6008 // x86 address has 4 operands: base, index, scale, and displacement 6009 int lastAddrIndx = 3; // [0,3] 6010 int valArgIndx = 4; 6011 6012 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6013 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6014 for (int i=0; i <= lastAddrIndx; ++i) 6015 (*MIB).addOperand(*argOpers[i]); 6016 6017 unsigned tt = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6018 if (invSrc) { 6019 MIB = BuildMI(newMBB, TII->get(X86::NOT32r), tt).addReg(t1); 6020 } 6021 else 6022 tt = t1; 6023 6024 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6025 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6026 && "invalid operand"); 6027 if (argOpers[valArgIndx]->isReg()) 6028 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6029 else 6030 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6031 MIB.addReg(tt); 6032 (*MIB).addOperand(*argOpers[valArgIndx]); 6033 6034 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6035 MIB.addReg(t1); 6036 6037 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6038 for (int i=0; i <= lastAddrIndx; ++i) 6039 (*MIB).addOperand(*argOpers[i]); 6040 MIB.addReg(t2); 6041 6042 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6043 MIB.addReg(X86::EAX); 6044 6045 // insert branch 6046 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6047 6048 delete bInstr; // The pseudo instruction is gone now. 6049 return nextMBB; 6050} 6051 6052// private utility function 6053MachineBasicBlock * 6054X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6055 MachineBasicBlock *MBB, 6056 unsigned cmovOpc) { 6057 // For the atomic min/max operator, we generate 6058 // thisMBB: 6059 // newMBB: 6060 // ld t1 = [min/max.addr] 6061 // mov t2 = [min/max.val] 6062 // cmp t1, t2 6063 // cmov[cond] t2 = t1 6064 // mov EAX = t1 6065 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6066 // bz newMBB 6067 // fallthrough -->nextMBB 6068 // 6069 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6070 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6071 ilist<MachineBasicBlock>::iterator MBBIter = MBB; 6072 ++MBBIter; 6073 6074 /// First build the CFG 6075 MachineFunction *F = MBB->getParent(); 6076 MachineBasicBlock *thisMBB = MBB; 6077 MachineBasicBlock *newMBB = new MachineBasicBlock(LLVM_BB); 6078 MachineBasicBlock *nextMBB = new MachineBasicBlock(LLVM_BB); 6079 F->getBasicBlockList().insert(MBBIter, newMBB); 6080 F->getBasicBlockList().insert(MBBIter, nextMBB); 6081 6082 // Move all successors to thisMBB to nextMBB 6083 nextMBB->transferSuccessors(thisMBB); 6084 6085 // Update thisMBB to fall through to newMBB 6086 thisMBB->addSuccessor(newMBB); 6087 6088 // newMBB jumps to newMBB and fall through to nextMBB 6089 newMBB->addSuccessor(nextMBB); 6090 newMBB->addSuccessor(newMBB); 6091 6092 // Insert instructions into newMBB based on incoming instruction 6093 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6094 MachineOperand& destOper = mInstr->getOperand(0); 6095 MachineOperand* argOpers[6]; 6096 int numArgs = mInstr->getNumOperands() - 1; 6097 for (int i=0; i < numArgs; ++i) 6098 argOpers[i] = &mInstr->getOperand(i+1); 6099 6100 // x86 address has 4 operands: base, index, scale, and displacement 6101 int lastAddrIndx = 3; // [0,3] 6102 int valArgIndx = 4; 6103 6104 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6105 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6106 for (int i=0; i <= lastAddrIndx; ++i) 6107 (*MIB).addOperand(*argOpers[i]); 6108 6109 // We only support register and immediate values 6110 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6111 && "invalid operand"); 6112 6113 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6114 if (argOpers[valArgIndx]->isReg()) 6115 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6116 else 6117 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6118 (*MIB).addOperand(*argOpers[valArgIndx]); 6119 6120 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6121 MIB.addReg(t1); 6122 6123 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6124 MIB.addReg(t1); 6125 MIB.addReg(t2); 6126 6127 // Generate movc 6128 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6129 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6130 MIB.addReg(t2); 6131 MIB.addReg(t1); 6132 6133 // Cmp and exchange if none has modified the memory location 6134 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6135 for (int i=0; i <= lastAddrIndx; ++i) 6136 (*MIB).addOperand(*argOpers[i]); 6137 MIB.addReg(t3); 6138 6139 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6140 MIB.addReg(X86::EAX); 6141 6142 // insert branch 6143 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6144 6145 delete mInstr; // The pseudo instruction is gone now. 6146 return nextMBB; 6147} 6148 6149 6150MachineBasicBlock * 6151X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6152 MachineBasicBlock *BB) { 6153 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6154 switch (MI->getOpcode()) { 6155 default: assert(false && "Unexpected instr type to insert"); 6156 case X86::CMOV_FR32: 6157 case X86::CMOV_FR64: 6158 case X86::CMOV_V4F32: 6159 case X86::CMOV_V2F64: 6160 case X86::CMOV_V2I64: { 6161 // To "insert" a SELECT_CC instruction, we actually have to insert the 6162 // diamond control-flow pattern. The incoming instruction knows the 6163 // destination vreg to set, the condition code register to branch on, the 6164 // true/false values to select between, and a branch opcode to use. 6165 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6166 ilist<MachineBasicBlock>::iterator It = BB; 6167 ++It; 6168 6169 // thisMBB: 6170 // ... 6171 // TrueVal = ... 6172 // cmpTY ccX, r1, r2 6173 // bCC copy1MBB 6174 // fallthrough --> copy0MBB 6175 MachineBasicBlock *thisMBB = BB; 6176 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB); 6177 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB); 6178 unsigned Opc = 6179 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6180 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6181 MachineFunction *F = BB->getParent(); 6182 F->getBasicBlockList().insert(It, copy0MBB); 6183 F->getBasicBlockList().insert(It, sinkMBB); 6184 // Update machine-CFG edges by transferring all successors of the current 6185 // block to the new block which will contain the Phi node for the select. 6186 sinkMBB->transferSuccessors(BB); 6187 6188 // Add the true and fallthrough blocks as its successors. 6189 BB->addSuccessor(copy0MBB); 6190 BB->addSuccessor(sinkMBB); 6191 6192 // copy0MBB: 6193 // %FalseValue = ... 6194 // # fallthrough to sinkMBB 6195 BB = copy0MBB; 6196 6197 // Update machine-CFG edges 6198 BB->addSuccessor(sinkMBB); 6199 6200 // sinkMBB: 6201 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6202 // ... 6203 BB = sinkMBB; 6204 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6205 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6206 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6207 6208 delete MI; // The pseudo instruction is gone now. 6209 return BB; 6210 } 6211 6212 case X86::FP32_TO_INT16_IN_MEM: 6213 case X86::FP32_TO_INT32_IN_MEM: 6214 case X86::FP32_TO_INT64_IN_MEM: 6215 case X86::FP64_TO_INT16_IN_MEM: 6216 case X86::FP64_TO_INT32_IN_MEM: 6217 case X86::FP64_TO_INT64_IN_MEM: 6218 case X86::FP80_TO_INT16_IN_MEM: 6219 case X86::FP80_TO_INT32_IN_MEM: 6220 case X86::FP80_TO_INT64_IN_MEM: { 6221 // Change the floating point control register to use "round towards zero" 6222 // mode when truncating to an integer value. 6223 MachineFunction *F = BB->getParent(); 6224 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6225 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6226 6227 // Load the old value of the high byte of the control word... 6228 unsigned OldCW = 6229 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6230 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6231 6232 // Set the high part to be round to zero... 6233 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6234 .addImm(0xC7F); 6235 6236 // Reload the modified control word now... 6237 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6238 6239 // Restore the memory image of control word to original value 6240 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6241 .addReg(OldCW); 6242 6243 // Get the X86 opcode to use. 6244 unsigned Opc; 6245 switch (MI->getOpcode()) { 6246 default: assert(0 && "illegal opcode!"); 6247 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6248 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6249 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6250 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6251 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6252 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6253 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6254 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6255 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6256 } 6257 6258 X86AddressMode AM; 6259 MachineOperand &Op = MI->getOperand(0); 6260 if (Op.isRegister()) { 6261 AM.BaseType = X86AddressMode::RegBase; 6262 AM.Base.Reg = Op.getReg(); 6263 } else { 6264 AM.BaseType = X86AddressMode::FrameIndexBase; 6265 AM.Base.FrameIndex = Op.getIndex(); 6266 } 6267 Op = MI->getOperand(1); 6268 if (Op.isImmediate()) 6269 AM.Scale = Op.getImm(); 6270 Op = MI->getOperand(2); 6271 if (Op.isImmediate()) 6272 AM.IndexReg = Op.getImm(); 6273 Op = MI->getOperand(3); 6274 if (Op.isGlobalAddress()) { 6275 AM.GV = Op.getGlobal(); 6276 } else { 6277 AM.Disp = Op.getImm(); 6278 } 6279 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6280 .addReg(MI->getOperand(4).getReg()); 6281 6282 // Reload the original control word now. 6283 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6284 6285 delete MI; // The pseudo instruction is gone now. 6286 return BB; 6287 } 6288 case X86::ATOMAND32: 6289 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6290 X86::AND32ri); 6291 case X86::ATOMOR32: 6292 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6293 X86::OR32ri); 6294 case X86::ATOMXOR32: 6295 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6296 X86::XOR32ri); 6297 case X86::ATOMNAND32: 6298 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6299 X86::AND32ri, true); 6300 case X86::ATOMMIN32: 6301 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6302 case X86::ATOMMAX32: 6303 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6304 case X86::ATOMUMIN32: 6305 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6306 case X86::ATOMUMAX32: 6307 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6308 } 6309} 6310 6311//===----------------------------------------------------------------------===// 6312// X86 Optimization Hooks 6313//===----------------------------------------------------------------------===// 6314 6315void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op, 6316 const APInt &Mask, 6317 APInt &KnownZero, 6318 APInt &KnownOne, 6319 const SelectionDAG &DAG, 6320 unsigned Depth) const { 6321 unsigned Opc = Op.getOpcode(); 6322 assert((Opc >= ISD::BUILTIN_OP_END || 6323 Opc == ISD::INTRINSIC_WO_CHAIN || 6324 Opc == ISD::INTRINSIC_W_CHAIN || 6325 Opc == ISD::INTRINSIC_VOID) && 6326 "Should use MaskedValueIsZero if you don't know whether Op" 6327 " is a target node!"); 6328 6329 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6330 switch (Opc) { 6331 default: break; 6332 case X86ISD::SETCC: 6333 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6334 Mask.getBitWidth() - 1); 6335 break; 6336 } 6337} 6338 6339/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6340/// node is a GlobalAddress + offset. 6341bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6342 GlobalValue* &GA, int64_t &Offset) const{ 6343 if (N->getOpcode() == X86ISD::Wrapper) { 6344 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6345 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6346 return true; 6347 } 6348 } 6349 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6350} 6351 6352static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6353 const TargetLowering &TLI) { 6354 GlobalValue *GV; 6355 int64_t Offset = 0; 6356 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6357 return (GV->getAlignment() >= N && (Offset % N) == 0); 6358 // DAG combine handles the stack object case. 6359 return false; 6360} 6361 6362static bool EltsFromConsecutiveLoads(SDNode *N, SDOperand PermMask, 6363 unsigned NumElems, MVT EVT, 6364 SDNode *&Base, 6365 SelectionDAG &DAG, MachineFrameInfo *MFI, 6366 const TargetLowering &TLI) { 6367 Base = NULL; 6368 for (unsigned i = 0; i < NumElems; ++i) { 6369 SDOperand Idx = PermMask.getOperand(i); 6370 if (Idx.getOpcode() == ISD::UNDEF) { 6371 if (!Base) 6372 return false; 6373 continue; 6374 } 6375 6376 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 6377 SDOperand Elt = DAG.getShuffleScalarElt(N, Index); 6378 if (!Elt.Val || 6379 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.Val))) 6380 return false; 6381 if (!Base) { 6382 Base = Elt.Val; 6383 if (Base->getOpcode() == ISD::UNDEF) 6384 return false; 6385 continue; 6386 } 6387 if (Elt.getOpcode() == ISD::UNDEF) 6388 continue; 6389 6390 if (!TLI.isConsecutiveLoad(Elt.Val, Base, 6391 EVT.getSizeInBits()/8, i, MFI)) 6392 return false; 6393 } 6394 return true; 6395} 6396 6397/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6398/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6399/// if the load addresses are consecutive, non-overlapping, and in the right 6400/// order. 6401static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6402 const TargetLowering &TLI) { 6403 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6404 MVT VT = N->getValueType(0); 6405 MVT EVT = VT.getVectorElementType(); 6406 SDOperand PermMask = N->getOperand(2); 6407 unsigned NumElems = PermMask.getNumOperands(); 6408 SDNode *Base = NULL; 6409 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6410 DAG, MFI, TLI)) 6411 return SDOperand(); 6412 6413 LoadSDNode *LD = cast<LoadSDNode>(Base); 6414 if (isBaseAlignmentOfN(16, Base->getOperand(1).Val, TLI)) 6415 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6416 LD->getSrcValueOffset(), LD->isVolatile()); 6417 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6418 LD->getSrcValueOffset(), LD->isVolatile(), 6419 LD->getAlignment()); 6420} 6421 6422/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6423static SDOperand PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6424 const X86Subtarget *Subtarget, 6425 const TargetLowering &TLI) { 6426 unsigned NumOps = N->getNumOperands(); 6427 6428 // Ignore single operand BUILD_VECTOR. 6429 if (NumOps == 1) 6430 return SDOperand(); 6431 6432 MVT VT = N->getValueType(0); 6433 MVT EVT = VT.getVectorElementType(); 6434 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6435 // We are looking for load i64 and zero extend. We want to transform 6436 // it before legalizer has a chance to expand it. Also look for i64 6437 // BUILD_PAIR bit casted to f64. 6438 return SDOperand(); 6439 // This must be an insertion into a zero vector. 6440 SDOperand HighElt = N->getOperand(1); 6441 if (!isZeroNode(HighElt)) 6442 return SDOperand(); 6443 6444 // Value must be a load. 6445 SDNode *Base = N->getOperand(0).Val; 6446 if (!isa<LoadSDNode>(Base)) { 6447 if (Base->getOpcode() != ISD::BIT_CONVERT) 6448 return SDOperand(); 6449 Base = Base->getOperand(0).Val; 6450 if (!isa<LoadSDNode>(Base)) 6451 return SDOperand(); 6452 } 6453 6454 // Transform it into VZEXT_LOAD addr. 6455 LoadSDNode *LD = cast<LoadSDNode>(Base); 6456 6457 // Load must not be an extload. 6458 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6459 return SDOperand(); 6460 6461 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6462} 6463 6464/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6465static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6466 const X86Subtarget *Subtarget) { 6467 SDOperand Cond = N->getOperand(0); 6468 6469 // If we have SSE[12] support, try to form min/max nodes. 6470 if (Subtarget->hasSSE2() && 6471 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6472 if (Cond.getOpcode() == ISD::SETCC) { 6473 // Get the LHS/RHS of the select. 6474 SDOperand LHS = N->getOperand(1); 6475 SDOperand RHS = N->getOperand(2); 6476 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6477 6478 unsigned Opcode = 0; 6479 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6480 switch (CC) { 6481 default: break; 6482 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6483 case ISD::SETULE: 6484 case ISD::SETLE: 6485 if (!UnsafeFPMath) break; 6486 // FALL THROUGH. 6487 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6488 case ISD::SETLT: 6489 Opcode = X86ISD::FMIN; 6490 break; 6491 6492 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6493 case ISD::SETUGT: 6494 case ISD::SETGT: 6495 if (!UnsafeFPMath) break; 6496 // FALL THROUGH. 6497 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6498 case ISD::SETGE: 6499 Opcode = X86ISD::FMAX; 6500 break; 6501 } 6502 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6503 switch (CC) { 6504 default: break; 6505 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6506 case ISD::SETUGT: 6507 case ISD::SETGT: 6508 if (!UnsafeFPMath) break; 6509 // FALL THROUGH. 6510 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6511 case ISD::SETGE: 6512 Opcode = X86ISD::FMIN; 6513 break; 6514 6515 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6516 case ISD::SETULE: 6517 case ISD::SETLE: 6518 if (!UnsafeFPMath) break; 6519 // FALL THROUGH. 6520 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6521 case ISD::SETLT: 6522 Opcode = X86ISD::FMAX; 6523 break; 6524 } 6525 } 6526 6527 if (Opcode) 6528 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6529 } 6530 6531 } 6532 6533 return SDOperand(); 6534} 6535 6536/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6537static SDOperand PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6538 const X86Subtarget *Subtarget) { 6539 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6540 // the FP state in cases where an emms may be missing. 6541 // A preferable solution to the general problem is to figure out the right 6542 // places to insert EMMS. This qualifies as a quick hack. 6543 StoreSDNode *St = cast<StoreSDNode>(N); 6544 if (St->getValue().getValueType().isVector() && 6545 St->getValue().getValueType().getSizeInBits() == 64 && 6546 isa<LoadSDNode>(St->getValue()) && 6547 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6548 St->getChain().hasOneUse() && !St->isVolatile()) { 6549 SDNode* LdVal = St->getValue().Val; 6550 LoadSDNode *Ld = 0; 6551 int TokenFactorIndex = -1; 6552 SmallVector<SDOperand, 8> Ops; 6553 SDNode* ChainVal = St->getChain().Val; 6554 // Must be a store of a load. We currently handle two cases: the load 6555 // is a direct child, and it's under an intervening TokenFactor. It is 6556 // possible to dig deeper under nested TokenFactors. 6557 if (ChainVal == LdVal) 6558 Ld = cast<LoadSDNode>(St->getChain()); 6559 else if (St->getValue().hasOneUse() && 6560 ChainVal->getOpcode() == ISD::TokenFactor) { 6561 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6562 if (ChainVal->getOperand(i).Val == LdVal) { 6563 TokenFactorIndex = i; 6564 Ld = cast<LoadSDNode>(St->getValue()); 6565 } else 6566 Ops.push_back(ChainVal->getOperand(i)); 6567 } 6568 } 6569 if (Ld) { 6570 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6571 if (Subtarget->is64Bit()) { 6572 SDOperand NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6573 Ld->getBasePtr(), Ld->getSrcValue(), 6574 Ld->getSrcValueOffset(), Ld->isVolatile(), 6575 Ld->getAlignment()); 6576 SDOperand NewChain = NewLd.getValue(1); 6577 if (TokenFactorIndex != -1) { 6578 Ops.push_back(NewChain); 6579 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6580 Ops.size()); 6581 } 6582 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6583 St->getSrcValue(), St->getSrcValueOffset(), 6584 St->isVolatile(), St->getAlignment()); 6585 } 6586 6587 // Otherwise, lower to two 32-bit copies. 6588 SDOperand LoAddr = Ld->getBasePtr(); 6589 SDOperand HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6590 DAG.getConstant(4, MVT::i32)); 6591 6592 SDOperand LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6593 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6594 Ld->isVolatile(), Ld->getAlignment()); 6595 SDOperand HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6596 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6597 Ld->isVolatile(), 6598 MinAlign(Ld->getAlignment(), 4)); 6599 6600 SDOperand NewChain = LoLd.getValue(1); 6601 if (TokenFactorIndex != -1) { 6602 Ops.push_back(LoLd); 6603 Ops.push_back(HiLd); 6604 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6605 Ops.size()); 6606 } 6607 6608 LoAddr = St->getBasePtr(); 6609 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6610 DAG.getConstant(4, MVT::i32)); 6611 6612 SDOperand LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6613 St->getSrcValue(), St->getSrcValueOffset(), 6614 St->isVolatile(), St->getAlignment()); 6615 SDOperand HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6616 St->getSrcValue(), St->getSrcValueOffset()+4, 6617 St->isVolatile(), 6618 MinAlign(St->getAlignment(), 4)); 6619 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6620 } 6621 } 6622 return SDOperand(); 6623} 6624 6625/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6626/// X86ISD::FXOR nodes. 6627static SDOperand PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6628 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6629 // F[X]OR(0.0, x) -> x 6630 // F[X]OR(x, 0.0) -> x 6631 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6632 if (C->getValueAPF().isPosZero()) 6633 return N->getOperand(1); 6634 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6635 if (C->getValueAPF().isPosZero()) 6636 return N->getOperand(0); 6637 return SDOperand(); 6638} 6639 6640/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 6641static SDOperand PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 6642 // FAND(0.0, x) -> 0.0 6643 // FAND(x, 0.0) -> 0.0 6644 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6645 if (C->getValueAPF().isPosZero()) 6646 return N->getOperand(0); 6647 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6648 if (C->getValueAPF().isPosZero()) 6649 return N->getOperand(1); 6650 return SDOperand(); 6651} 6652 6653 6654SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N, 6655 DAGCombinerInfo &DCI) const { 6656 SelectionDAG &DAG = DCI.DAG; 6657 switch (N->getOpcode()) { 6658 default: break; 6659 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 6660 case ISD::BUILD_VECTOR: 6661 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 6662 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 6663 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 6664 case X86ISD::FXOR: 6665 case X86ISD::FOR: return PerformFORCombine(N, DAG); 6666 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 6667 } 6668 6669 return SDOperand(); 6670} 6671 6672//===----------------------------------------------------------------------===// 6673// X86 Inline Assembly Support 6674//===----------------------------------------------------------------------===// 6675 6676/// getConstraintType - Given a constraint letter, return the type of 6677/// constraint it is for this target. 6678X86TargetLowering::ConstraintType 6679X86TargetLowering::getConstraintType(const std::string &Constraint) const { 6680 if (Constraint.size() == 1) { 6681 switch (Constraint[0]) { 6682 case 'A': 6683 case 'f': 6684 case 'r': 6685 case 'R': 6686 case 'l': 6687 case 'q': 6688 case 'Q': 6689 case 'x': 6690 case 'y': 6691 case 'Y': 6692 return C_RegisterClass; 6693 default: 6694 break; 6695 } 6696 } 6697 return TargetLowering::getConstraintType(Constraint); 6698} 6699 6700/// LowerXConstraint - try to replace an X constraint, which matches anything, 6701/// with another that has more specific requirements based on the type of the 6702/// corresponding operand. 6703const char *X86TargetLowering:: 6704LowerXConstraint(MVT ConstraintVT) const { 6705 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 6706 // 'f' like normal targets. 6707 if (ConstraintVT.isFloatingPoint()) { 6708 if (Subtarget->hasSSE2()) 6709 return "Y"; 6710 if (Subtarget->hasSSE1()) 6711 return "x"; 6712 } 6713 6714 return TargetLowering::LowerXConstraint(ConstraintVT); 6715} 6716 6717/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 6718/// vector. If it is invalid, don't add anything to Ops. 6719void X86TargetLowering::LowerAsmOperandForConstraint(SDOperand Op, 6720 char Constraint, 6721 std::vector<SDOperand>&Ops, 6722 SelectionDAG &DAG) const { 6723 SDOperand Result(0, 0); 6724 6725 switch (Constraint) { 6726 default: break; 6727 case 'I': 6728 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6729 if (C->getValue() <= 31) { 6730 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6731 break; 6732 } 6733 } 6734 return; 6735 case 'N': 6736 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6737 if (C->getValue() <= 255) { 6738 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6739 break; 6740 } 6741 } 6742 return; 6743 case 'i': { 6744 // Literal immediates are always ok. 6745 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 6746 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 6747 break; 6748 } 6749 6750 // If we are in non-pic codegen mode, we allow the address of a global (with 6751 // an optional displacement) to be used with 'i'. 6752 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 6753 int64_t Offset = 0; 6754 6755 // Match either (GA) or (GA+C) 6756 if (GA) { 6757 Offset = GA->getOffset(); 6758 } else if (Op.getOpcode() == ISD::ADD) { 6759 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6760 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6761 if (C && GA) { 6762 Offset = GA->getOffset()+C->getValue(); 6763 } else { 6764 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6765 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 6766 if (C && GA) 6767 Offset = GA->getOffset()+C->getValue(); 6768 else 6769 C = 0, GA = 0; 6770 } 6771 } 6772 6773 if (GA) { 6774 // If addressing this global requires a load (e.g. in PIC mode), we can't 6775 // match. 6776 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 6777 false)) 6778 return; 6779 6780 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 6781 Offset); 6782 Result = Op; 6783 break; 6784 } 6785 6786 // Otherwise, not valid for this mode. 6787 return; 6788 } 6789 } 6790 6791 if (Result.Val) { 6792 Ops.push_back(Result); 6793 return; 6794 } 6795 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 6796} 6797 6798std::vector<unsigned> X86TargetLowering:: 6799getRegClassForInlineAsmConstraint(const std::string &Constraint, 6800 MVT VT) const { 6801 if (Constraint.size() == 1) { 6802 // FIXME: not handling fp-stack yet! 6803 switch (Constraint[0]) { // GCC X86 Constraint Letters 6804 default: break; // Unknown constraint letter 6805 case 'A': // EAX/EDX 6806 if (VT == MVT::i32 || VT == MVT::i64) 6807 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 6808 break; 6809 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 6810 case 'Q': // Q_REGS 6811 if (VT == MVT::i32) 6812 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 6813 else if (VT == MVT::i16) 6814 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 6815 else if (VT == MVT::i8) 6816 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 6817 else if (VT == MVT::i64) 6818 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 6819 break; 6820 } 6821 } 6822 6823 return std::vector<unsigned>(); 6824} 6825 6826std::pair<unsigned, const TargetRegisterClass*> 6827X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 6828 MVT VT) const { 6829 // First, see if this is a constraint that directly corresponds to an LLVM 6830 // register class. 6831 if (Constraint.size() == 1) { 6832 // GCC Constraint Letters 6833 switch (Constraint[0]) { 6834 default: break; 6835 case 'r': // GENERAL_REGS 6836 case 'R': // LEGACY_REGS 6837 case 'l': // INDEX_REGS 6838 if (VT == MVT::i64 && Subtarget->is64Bit()) 6839 return std::make_pair(0U, X86::GR64RegisterClass); 6840 if (VT == MVT::i32) 6841 return std::make_pair(0U, X86::GR32RegisterClass); 6842 else if (VT == MVT::i16) 6843 return std::make_pair(0U, X86::GR16RegisterClass); 6844 else if (VT == MVT::i8) 6845 return std::make_pair(0U, X86::GR8RegisterClass); 6846 break; 6847 case 'f': // FP Stack registers. 6848 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 6849 // value to the correct fpstack register class. 6850 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 6851 return std::make_pair(0U, X86::RFP32RegisterClass); 6852 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 6853 return std::make_pair(0U, X86::RFP64RegisterClass); 6854 return std::make_pair(0U, X86::RFP80RegisterClass); 6855 case 'y': // MMX_REGS if MMX allowed. 6856 if (!Subtarget->hasMMX()) break; 6857 return std::make_pair(0U, X86::VR64RegisterClass); 6858 break; 6859 case 'Y': // SSE_REGS if SSE2 allowed 6860 if (!Subtarget->hasSSE2()) break; 6861 // FALL THROUGH. 6862 case 'x': // SSE_REGS if SSE1 allowed 6863 if (!Subtarget->hasSSE1()) break; 6864 6865 switch (VT.getSimpleVT()) { 6866 default: break; 6867 // Scalar SSE types. 6868 case MVT::f32: 6869 case MVT::i32: 6870 return std::make_pair(0U, X86::FR32RegisterClass); 6871 case MVT::f64: 6872 case MVT::i64: 6873 return std::make_pair(0U, X86::FR64RegisterClass); 6874 // Vector types. 6875 case MVT::v16i8: 6876 case MVT::v8i16: 6877 case MVT::v4i32: 6878 case MVT::v2i64: 6879 case MVT::v4f32: 6880 case MVT::v2f64: 6881 return std::make_pair(0U, X86::VR128RegisterClass); 6882 } 6883 break; 6884 } 6885 } 6886 6887 // Use the default implementation in TargetLowering to convert the register 6888 // constraint into a member of a register class. 6889 std::pair<unsigned, const TargetRegisterClass*> Res; 6890 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 6891 6892 // Not found as a standard register? 6893 if (Res.second == 0) { 6894 // GCC calls "st(0)" just plain "st". 6895 if (StringsEqualNoCase("{st}", Constraint)) { 6896 Res.first = X86::ST0; 6897 Res.second = X86::RFP80RegisterClass; 6898 } 6899 6900 return Res; 6901 } 6902 6903 // Otherwise, check to see if this is a register class of the wrong value 6904 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 6905 // turn into {ax},{dx}. 6906 if (Res.second->hasType(VT)) 6907 return Res; // Correct type already, nothing to do. 6908 6909 // All of the single-register GCC register classes map their values onto 6910 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 6911 // really want an 8-bit or 32-bit register, map to the appropriate register 6912 // class and return the appropriate register. 6913 if (Res.second != X86::GR16RegisterClass) 6914 return Res; 6915 6916 if (VT == MVT::i8) { 6917 unsigned DestReg = 0; 6918 switch (Res.first) { 6919 default: break; 6920 case X86::AX: DestReg = X86::AL; break; 6921 case X86::DX: DestReg = X86::DL; break; 6922 case X86::CX: DestReg = X86::CL; break; 6923 case X86::BX: DestReg = X86::BL; break; 6924 } 6925 if (DestReg) { 6926 Res.first = DestReg; 6927 Res.second = Res.second = X86::GR8RegisterClass; 6928 } 6929 } else if (VT == MVT::i32) { 6930 unsigned DestReg = 0; 6931 switch (Res.first) { 6932 default: break; 6933 case X86::AX: DestReg = X86::EAX; break; 6934 case X86::DX: DestReg = X86::EDX; break; 6935 case X86::CX: DestReg = X86::ECX; break; 6936 case X86::BX: DestReg = X86::EBX; break; 6937 case X86::SI: DestReg = X86::ESI; break; 6938 case X86::DI: DestReg = X86::EDI; break; 6939 case X86::BP: DestReg = X86::EBP; break; 6940 case X86::SP: DestReg = X86::ESP; break; 6941 } 6942 if (DestReg) { 6943 Res.first = DestReg; 6944 Res.second = Res.second = X86::GR32RegisterClass; 6945 } 6946 } else if (VT == MVT::i64) { 6947 unsigned DestReg = 0; 6948 switch (Res.first) { 6949 default: break; 6950 case X86::AX: DestReg = X86::RAX; break; 6951 case X86::DX: DestReg = X86::RDX; break; 6952 case X86::CX: DestReg = X86::RCX; break; 6953 case X86::BX: DestReg = X86::RBX; break; 6954 case X86::SI: DestReg = X86::RSI; break; 6955 case X86::DI: DestReg = X86::RDI; break; 6956 case X86::BP: DestReg = X86::RBP; break; 6957 case X86::SP: DestReg = X86::RSP; break; 6958 } 6959 if (DestReg) { 6960 Res.first = DestReg; 6961 Res.second = Res.second = X86::GR64RegisterClass; 6962 } 6963 } 6964 6965 return Res; 6966} 6967