X86ISelLowering.cpp revision d9f3c480a7bc0969b08ace68af7dcde40f6caff1
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86TargetMachine.h" 20#include "llvm/CallingConv.h" 21#include "llvm/Constants.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/GlobalVariable.h" 24#include "llvm/Function.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/ADT/BitVector.h" 27#include "llvm/ADT/VectorExtras.h" 28#include "llvm/CodeGen/CallingConvLower.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/PseudoSourceValue.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/Debug.h" 38#include "llvm/Target/TargetOptions.h" 39#include "llvm/ADT/SmallSet.h" 40#include "llvm/ADT/StringExtras.h" 41using namespace llvm; 42 43// Forward declarations. 44static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45 46X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47 : TargetLowering(TM) { 48 Subtarget = &TM.getSubtarget<X86Subtarget>(); 49 X86ScalarSSEf64 = Subtarget->hasSSE2(); 50 X86ScalarSSEf32 = Subtarget->hasSSE1(); 51 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52 53 bool Fast = false; 54 55 RegInfo = TM.getRegisterInfo(); 56 57 // Set up the TargetLowering object. 58 59 // X86 is weird, it always uses i8 for shift amounts and setcc results. 60 setShiftAmountType(MVT::i8); 61 setSetCCResultContents(ZeroOrOneSetCCResult); 62 setSchedulingPreference(SchedulingForRegPressure); 63 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 64 setStackPointerRegisterToSaveRestore(X86StackPtr); 65 66 if (Subtarget->isTargetDarwin()) { 67 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 68 setUseUnderscoreSetJmp(false); 69 setUseUnderscoreLongJmp(false); 70 } else if (Subtarget->isTargetMingw()) { 71 // MS runtime is weird: it exports _setjmp, but longjmp! 72 setUseUnderscoreSetJmp(true); 73 setUseUnderscoreLongJmp(false); 74 } else { 75 setUseUnderscoreSetJmp(true); 76 setUseUnderscoreLongJmp(true); 77 } 78 79 // Set up the register classes. 80 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 81 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 82 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 83 if (Subtarget->is64Bit()) 84 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 85 86 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 87 88 // We don't accept any truncstore of integer registers. 89 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 90 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 91 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 92 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 93 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 94 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 95 96 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 97 // operation. 98 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 99 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 100 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 101 102 if (Subtarget->is64Bit()) { 103 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 104 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 105 } else { 106 if (X86ScalarSSEf64) 107 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 108 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 109 else 110 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 111 } 112 113 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 114 // this operation. 115 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 116 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 117 // SSE has no i16 to fp conversion, only i32 118 if (X86ScalarSSEf32) { 119 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 120 // f32 and f64 cases are Legal, f80 case is not 121 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 122 } else { 123 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 124 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 125 } 126 127 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 128 // are Legal, f80 is custom lowered. 129 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 130 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 131 132 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 133 // this operation. 134 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 135 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 136 137 if (X86ScalarSSEf32) { 138 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 139 // f32 and f64 cases are Legal, f80 case is not 140 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 141 } else { 142 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 143 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 144 } 145 146 // Handle FP_TO_UINT by promoting the destination to a larger signed 147 // conversion. 148 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 149 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 151 152 if (Subtarget->is64Bit()) { 153 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 155 } else { 156 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 157 // Expand FP_TO_UINT into a select. 158 // FIXME: We would like to use a Custom expander here eventually to do 159 // the optimal thing for SSE vs. the default expansion in the legalizer. 160 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 161 else 162 // With SSE3 we can use fisttpll to convert to a signed i64. 163 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 164 } 165 166 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 167 if (!X86ScalarSSEf64) { 168 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 169 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 170 } 171 172 // Scalar integer divide and remainder are lowered to use operations that 173 // produce two results, to match the available instructions. This exposes 174 // the two-result form to trivial CSE, which is able to combine x/y and x%y 175 // into a single instruction. 176 // 177 // Scalar integer multiply-high is also lowered to use two-result 178 // operations, to match the available instructions. However, plain multiply 179 // (low) operations are left as Legal, as there are single-result 180 // instructions for this in x86. Using the two-result multiply instructions 181 // when both high and low results are needed must be arranged by dagcombine. 182 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 183 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 184 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 185 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::SREM , MVT::i8 , Expand); 187 setOperationAction(ISD::UREM , MVT::i8 , Expand); 188 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 189 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 190 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 191 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::SREM , MVT::i16 , Expand); 193 setOperationAction(ISD::UREM , MVT::i16 , Expand); 194 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 195 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 196 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 197 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::SREM , MVT::i32 , Expand); 199 setOperationAction(ISD::UREM , MVT::i32 , Expand); 200 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 201 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 202 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 203 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::SREM , MVT::i64 , Expand); 205 setOperationAction(ISD::UREM , MVT::i64 , Expand); 206 207 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 208 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 209 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 210 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 211 if (Subtarget->is64Bit()) 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 216 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 217 setOperationAction(ISD::FREM , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f64 , Expand); 219 setOperationAction(ISD::FREM , MVT::f80 , Expand); 220 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 221 222 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 223 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 224 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 226 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 227 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 229 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 230 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 231 if (Subtarget->is64Bit()) { 232 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 233 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 234 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 235 } 236 237 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 238 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 239 240 // These should be promoted to a larger select which is supported. 241 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 242 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 243 // X86 wants to expand cmov itself. 244 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 245 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 246 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 249 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 252 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 255 if (Subtarget->is64Bit()) { 256 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 257 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 258 } 259 // X86 ret instruction may pop stack. 260 setOperationAction(ISD::RET , MVT::Other, Custom); 261 if (!Subtarget->is64Bit()) 262 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 263 264 // Darwin ABI issue. 265 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 266 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 268 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 269 if (Subtarget->is64Bit()) 270 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 271 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 272 if (Subtarget->is64Bit()) { 273 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 274 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 275 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 276 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 277 } 278 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 279 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 281 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 282 if (Subtarget->is64Bit()) { 283 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 285 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 286 } 287 288 if (Subtarget->hasSSE1()) 289 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 290 291 if (!Subtarget->hasSSE2()) 292 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 293 294 // Expand certain atomics 295 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i8, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i16, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i32, Custom); 298 setOperationAction(ISD::ATOMIC_CMP_SWAP , MVT::i64, Custom); 299 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i8, Expand); 300 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i16, Expand); 301 setOperationAction(ISD::ATOMIC_LOAD_SUB , MVT::i32, Expand); 302 303 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 304 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 305 // FIXME - use subtarget debug flags 306 if (!Subtarget->isTargetDarwin() && 307 !Subtarget->isTargetELF() && 308 !Subtarget->isTargetCygMing()) { 309 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 310 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 311 } 312 313 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 314 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 315 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 316 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 317 if (Subtarget->is64Bit()) { 318 // FIXME: Verify 319 setExceptionPointerRegister(X86::RAX); 320 setExceptionSelectorRegister(X86::RDX); 321 } else { 322 setExceptionPointerRegister(X86::EAX); 323 setExceptionSelectorRegister(X86::EDX); 324 } 325 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 326 327 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 328 329 setOperationAction(ISD::TRAP, MVT::Other, Legal); 330 331 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 332 setOperationAction(ISD::VASTART , MVT::Other, Custom); 333 setOperationAction(ISD::VAEND , MVT::Other, Expand); 334 if (Subtarget->is64Bit()) { 335 setOperationAction(ISD::VAARG , MVT::Other, Custom); 336 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 337 } else { 338 setOperationAction(ISD::VAARG , MVT::Other, Expand); 339 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 340 } 341 342 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 343 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 344 if (Subtarget->is64Bit()) 345 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 346 if (Subtarget->isTargetCygMing()) 347 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 348 else 349 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 350 351 if (X86ScalarSSEf64) { 352 // f32 and f64 use SSE. 353 // Set up the FP register classes. 354 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 355 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 356 357 // Use ANDPD to simulate FABS. 358 setOperationAction(ISD::FABS , MVT::f64, Custom); 359 setOperationAction(ISD::FABS , MVT::f32, Custom); 360 361 // Use XORP to simulate FNEG. 362 setOperationAction(ISD::FNEG , MVT::f64, Custom); 363 setOperationAction(ISD::FNEG , MVT::f32, Custom); 364 365 // Use ANDPD and ORPD to simulate FCOPYSIGN. 366 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 367 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 368 369 // We don't support sin/cos/fmod 370 setOperationAction(ISD::FSIN , MVT::f64, Expand); 371 setOperationAction(ISD::FCOS , MVT::f64, Expand); 372 setOperationAction(ISD::FSIN , MVT::f32, Expand); 373 setOperationAction(ISD::FCOS , MVT::f32, Expand); 374 375 // Expand FP immediates into loads from the stack, except for the special 376 // cases we handle. 377 addLegalFPImmediate(APFloat(+0.0)); // xorpd 378 addLegalFPImmediate(APFloat(+0.0f)); // xorps 379 380 // Floating truncations from f80 and extensions to f80 go through memory. 381 // If optimizing, we lie about this though and handle it in 382 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 383 if (Fast) { 384 setConvertAction(MVT::f32, MVT::f80, Expand); 385 setConvertAction(MVT::f64, MVT::f80, Expand); 386 setConvertAction(MVT::f80, MVT::f32, Expand); 387 setConvertAction(MVT::f80, MVT::f64, Expand); 388 } 389 } else if (X86ScalarSSEf32) { 390 // Use SSE for f32, x87 for f64. 391 // Set up the FP register classes. 392 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 393 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 394 395 // Use ANDPS to simulate FABS. 396 setOperationAction(ISD::FABS , MVT::f32, Custom); 397 398 // Use XORP to simulate FNEG. 399 setOperationAction(ISD::FNEG , MVT::f32, Custom); 400 401 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 402 403 // Use ANDPS and ORPS to simulate FCOPYSIGN. 404 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 405 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 406 407 // We don't support sin/cos/fmod 408 setOperationAction(ISD::FSIN , MVT::f32, Expand); 409 setOperationAction(ISD::FCOS , MVT::f32, Expand); 410 411 // Special cases we handle for FP constants. 412 addLegalFPImmediate(APFloat(+0.0f)); // xorps 413 addLegalFPImmediate(APFloat(+0.0)); // FLD0 414 addLegalFPImmediate(APFloat(+1.0)); // FLD1 415 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 416 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 417 418 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 419 // this though and handle it in InstructionSelectPreprocess so that 420 // dagcombine2 can hack on these. 421 if (Fast) { 422 setConvertAction(MVT::f32, MVT::f64, Expand); 423 setConvertAction(MVT::f32, MVT::f80, Expand); 424 setConvertAction(MVT::f80, MVT::f32, Expand); 425 setConvertAction(MVT::f64, MVT::f32, Expand); 426 // And x87->x87 truncations also. 427 setConvertAction(MVT::f80, MVT::f64, Expand); 428 } 429 430 if (!UnsafeFPMath) { 431 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 432 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 433 } 434 } else { 435 // f32 and f64 in x87. 436 // Set up the FP register classes. 437 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 438 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 439 440 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 441 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 442 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 443 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 444 445 // Floating truncations go through memory. If optimizing, we lie about 446 // this though and handle it in InstructionSelectPreprocess so that 447 // dagcombine2 can hack on these. 448 if (Fast) { 449 setConvertAction(MVT::f80, MVT::f32, Expand); 450 setConvertAction(MVT::f64, MVT::f32, Expand); 451 setConvertAction(MVT::f80, MVT::f64, Expand); 452 } 453 454 if (!UnsafeFPMath) { 455 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 456 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 457 } 458 addLegalFPImmediate(APFloat(+0.0)); // FLD0 459 addLegalFPImmediate(APFloat(+1.0)); // FLD1 460 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 461 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 462 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 463 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 464 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 465 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 466 } 467 468 // Long double always uses X87. 469 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 470 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 471 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 472 { 473 APFloat TmpFlt(+0.0); 474 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 475 addLegalFPImmediate(TmpFlt); // FLD0 476 TmpFlt.changeSign(); 477 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 478 APFloat TmpFlt2(+1.0); 479 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 480 addLegalFPImmediate(TmpFlt2); // FLD1 481 TmpFlt2.changeSign(); 482 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 483 } 484 485 if (!UnsafeFPMath) { 486 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 487 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 488 } 489 490 // Always use a library call for pow. 491 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 492 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 493 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 494 495 // First set operation action for all vector types to expand. Then we 496 // will selectively turn on ones that can be effectively codegen'd. 497 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 498 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 499 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 500 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 501 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 502 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 503 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 504 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 505 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 506 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 507 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 508 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 509 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 510 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 511 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 512 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 513 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 514 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::SimpleValueType)VT, Expand); 515 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 516 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 517 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 532 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 533 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 534 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 535 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 536 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 537 } 538 539 if (Subtarget->hasMMX()) { 540 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 541 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 542 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 543 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 544 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 545 546 // FIXME: add MMX packed arithmetics 547 548 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 549 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 550 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 551 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 552 553 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 554 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 555 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 556 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 557 558 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 559 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 560 561 setOperationAction(ISD::AND, MVT::v8i8, Promote); 562 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 563 setOperationAction(ISD::AND, MVT::v4i16, Promote); 564 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 565 setOperationAction(ISD::AND, MVT::v2i32, Promote); 566 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 567 setOperationAction(ISD::AND, MVT::v1i64, Legal); 568 569 setOperationAction(ISD::OR, MVT::v8i8, Promote); 570 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 571 setOperationAction(ISD::OR, MVT::v4i16, Promote); 572 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 573 setOperationAction(ISD::OR, MVT::v2i32, Promote); 574 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 575 setOperationAction(ISD::OR, MVT::v1i64, Legal); 576 577 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 578 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 579 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 580 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 581 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 582 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 583 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 584 585 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 586 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 587 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 588 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 589 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 590 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 591 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 592 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 593 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 594 595 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 596 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 597 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 598 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 599 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 600 601 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 602 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 603 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 604 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 605 606 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); 607 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 608 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 609 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 610 611 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 612 } 613 614 if (Subtarget->hasSSE1()) { 615 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 616 617 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 618 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 619 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 620 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 621 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 622 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 623 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 624 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 625 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 626 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 627 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 628 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 629 } 630 631 if (Subtarget->hasSSE2()) { 632 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 633 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 634 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 635 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 636 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 637 638 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 639 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 640 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 641 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 642 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 643 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 644 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 645 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 646 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 647 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 648 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 649 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 650 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 651 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 652 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 653 654 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 655 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 656 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 657 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 658 659 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 660 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 661 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 662 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 663 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 664 665 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 666 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 667 MVT VT = (MVT::SimpleValueType)i; 668 // Do not attempt to custom lower non-power-of-2 vectors 669 if (!isPowerOf2_32(VT.getVectorNumElements())) 670 continue; 671 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 672 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 673 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 674 } 675 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 676 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 677 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 678 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 679 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 680 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 681 if (Subtarget->is64Bit()) { 682 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 683 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 684 } 685 686 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 687 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 688 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 689 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 690 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 691 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 692 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 693 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 694 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 695 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 696 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 697 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 698 } 699 700 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 701 702 // Custom lower v2i64 and v2f64 selects. 703 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 704 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 705 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 706 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 707 708 } 709 710 if (Subtarget->hasSSE41()) { 711 // FIXME: Do we need to handle scalar-to-vector here? 712 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 713 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 714 715 // i8 and i16 vectors are custom , because the source register and source 716 // source memory operand types are not the same width. f32 vectors are 717 // custom since the immediate controlling the insert encodes additional 718 // information. 719 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 720 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 721 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 722 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 723 724 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 725 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 726 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 727 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 728 729 if (Subtarget->is64Bit()) { 730 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 731 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 732 } 733 } 734 735 if (Subtarget->hasSSE42()) { 736 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 737 } 738 739 // We want to custom lower some of our intrinsics. 740 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 741 742 // We have target-specific dag combine patterns for the following nodes: 743 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 744 setTargetDAGCombine(ISD::BUILD_VECTOR); 745 setTargetDAGCombine(ISD::SELECT); 746 setTargetDAGCombine(ISD::STORE); 747 748 computeRegisterProperties(); 749 750 // FIXME: These should be based on subtarget info. Plus, the values should 751 // be smaller when we are in optimizing for size mode. 752 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 753 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 754 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 755 allowUnalignedMemoryAccesses = true; // x86 supports it! 756 setPrefLoopAlignment(16); 757} 758 759 760MVT X86TargetLowering::getSetCCResultType(const SDValue &) const { 761 return MVT::i8; 762} 763 764 765/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 766/// the desired ByVal argument alignment. 767static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 768 if (MaxAlign == 16) 769 return; 770 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 771 if (VTy->getBitWidth() == 128) 772 MaxAlign = 16; 773 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 774 unsigned EltAlign = 0; 775 getMaxByValAlign(ATy->getElementType(), EltAlign); 776 if (EltAlign > MaxAlign) 777 MaxAlign = EltAlign; 778 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 779 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 780 unsigned EltAlign = 0; 781 getMaxByValAlign(STy->getElementType(i), EltAlign); 782 if (EltAlign > MaxAlign) 783 MaxAlign = EltAlign; 784 if (MaxAlign == 16) 785 break; 786 } 787 } 788 return; 789} 790 791/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 792/// function arguments in the caller parameter area. For X86, aggregates 793/// that contain SSE vectors are placed at 16-byte boundaries while the rest 794/// are at 4-byte boundaries. 795unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 796 if (Subtarget->is64Bit()) 797 return getTargetData()->getABITypeAlignment(Ty); 798 unsigned Align = 4; 799 if (Subtarget->hasSSE1()) 800 getMaxByValAlign(Ty, Align); 801 return Align; 802} 803 804/// getOptimalMemOpType - Returns the target specific optimal type for load 805/// and store operations as a result of memset, memcpy, and memmove 806/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 807/// determining it. 808MVT 809X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 810 bool isSrcConst, bool isSrcStr) const { 811 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 812 return MVT::v4i32; 813 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 814 return MVT::v4f32; 815 if (Subtarget->is64Bit() && Size >= 8) 816 return MVT::i64; 817 return MVT::i32; 818} 819 820 821/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 822/// jumptable. 823SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 824 SelectionDAG &DAG) const { 825 if (usesGlobalOffsetTable()) 826 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 827 if (!Subtarget->isPICStyleRIPRel()) 828 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 829 return Table; 830} 831 832//===----------------------------------------------------------------------===// 833// Return Value Calling Convention Implementation 834//===----------------------------------------------------------------------===// 835 836#include "X86GenCallingConv.inc" 837 838/// LowerRET - Lower an ISD::RET node. 839SDValue X86TargetLowering::LowerRET(SDValue Op, SelectionDAG &DAG) { 840 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 841 842 SmallVector<CCValAssign, 16> RVLocs; 843 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 844 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 845 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 846 CCInfo.AnalyzeReturn(Op.Val, RetCC_X86); 847 848 // If this is the first return lowered for this function, add the regs to the 849 // liveout set for the function. 850 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 851 for (unsigned i = 0; i != RVLocs.size(); ++i) 852 if (RVLocs[i].isRegLoc()) 853 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 854 } 855 SDValue Chain = Op.getOperand(0); 856 857 // Handle tail call return. 858 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 859 if (Chain.getOpcode() == X86ISD::TAILCALL) { 860 SDValue TailCall = Chain; 861 SDValue TargetAddress = TailCall.getOperand(1); 862 SDValue StackAdjustment = TailCall.getOperand(2); 863 assert(((TargetAddress.getOpcode() == ISD::Register && 864 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 865 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 866 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 867 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 868 "Expecting an global address, external symbol, or register"); 869 assert(StackAdjustment.getOpcode() == ISD::Constant && 870 "Expecting a const value"); 871 872 SmallVector<SDValue,8> Operands; 873 Operands.push_back(Chain.getOperand(0)); 874 Operands.push_back(TargetAddress); 875 Operands.push_back(StackAdjustment); 876 // Copy registers used by the call. Last operand is a flag so it is not 877 // copied. 878 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 879 Operands.push_back(Chain.getOperand(i)); 880 } 881 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 882 Operands.size()); 883 } 884 885 // Regular return. 886 SDValue Flag; 887 888 SmallVector<SDValue, 6> RetOps; 889 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 890 // Operand #1 = Bytes To Pop 891 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 892 893 // Copy the result values into the output registers. 894 for (unsigned i = 0; i != RVLocs.size(); ++i) { 895 CCValAssign &VA = RVLocs[i]; 896 assert(VA.isRegLoc() && "Can only return in registers!"); 897 SDValue ValToCopy = Op.getOperand(i*2+1); 898 899 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 900 // the RET instruction and handled by the FP Stackifier. 901 if (RVLocs[i].getLocReg() == X86::ST0 || 902 RVLocs[i].getLocReg() == X86::ST1) { 903 // If this is a copy from an xmm register to ST(0), use an FPExtend to 904 // change the value to the FP stack register class. 905 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 906 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 907 RetOps.push_back(ValToCopy); 908 // Don't emit a copytoreg. 909 continue; 910 } 911 912 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 913 Flag = Chain.getValue(1); 914 } 915 916 // The x86-64 ABI for returning structs by value requires that we copy 917 // the sret argument into %rax for the return. We saved the argument into 918 // a virtual register in the entry block, so now we copy the value out 919 // and into %rax. 920 if (Subtarget->is64Bit() && 921 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 922 MachineFunction &MF = DAG.getMachineFunction(); 923 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 924 unsigned Reg = FuncInfo->getSRetReturnReg(); 925 if (!Reg) { 926 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 927 FuncInfo->setSRetReturnReg(Reg); 928 } 929 SDValue Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 930 931 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 932 Flag = Chain.getValue(1); 933 } 934 935 RetOps[0] = Chain; // Update chain. 936 937 // Add the flag if we have it. 938 if (Flag.Val) 939 RetOps.push_back(Flag); 940 941 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 942} 943 944 945/// LowerCallResult - Lower the result values of an ISD::CALL into the 946/// appropriate copies out of appropriate physical registers. This assumes that 947/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 948/// being lowered. The returns a SDNode with the same number of values as the 949/// ISD::CALL. 950SDNode *X86TargetLowering:: 951LowerCallResult(SDValue Chain, SDValue InFlag, SDNode *TheCall, 952 unsigned CallingConv, SelectionDAG &DAG) { 953 954 // Assign locations to each value returned by this call. 955 SmallVector<CCValAssign, 16> RVLocs; 956 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 957 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 958 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 959 960 SmallVector<SDValue, 8> ResultVals; 961 962 // Copy all of the result registers out of their specified physreg. 963 for (unsigned i = 0; i != RVLocs.size(); ++i) { 964 MVT CopyVT = RVLocs[i].getValVT(); 965 966 // If this is a call to a function that returns an fp value on the floating 967 // point stack, but where we prefer to use the value in xmm registers, copy 968 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 969 if (RVLocs[i].getLocReg() == X86::ST0 && 970 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 971 CopyVT = MVT::f80; 972 } 973 974 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 975 CopyVT, InFlag).getValue(1); 976 SDValue Val = Chain.getValue(0); 977 InFlag = Chain.getValue(2); 978 979 if (CopyVT != RVLocs[i].getValVT()) { 980 // Round the F80 the right size, which also moves to the appropriate xmm 981 // register. 982 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 983 // This truncation won't change the value. 984 DAG.getIntPtrConstant(1)); 985 } 986 987 ResultVals.push_back(Val); 988 } 989 990 // Merge everything together with a MERGE_VALUES node. 991 ResultVals.push_back(Chain); 992 return DAG.getMergeValues(TheCall->getVTList(), &ResultVals[0], 993 ResultVals.size()).Val; 994} 995 996 997//===----------------------------------------------------------------------===// 998// C & StdCall & Fast Calling Convention implementation 999//===----------------------------------------------------------------------===// 1000// StdCall calling convention seems to be standard for many Windows' API 1001// routines and around. It differs from C calling convention just a little: 1002// callee should clean up the stack, not caller. Symbols should be also 1003// decorated in some fancy way :) It doesn't support any vector arguments. 1004// For info on fast calling convention see Fast Calling Convention (tail call) 1005// implementation LowerX86_32FastCCCallTo. 1006 1007/// AddLiveIn - This helper function adds the specified physical register to the 1008/// MachineFunction as a live in value. It also creates a corresponding virtual 1009/// register for it. 1010static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1011 const TargetRegisterClass *RC) { 1012 assert(RC->contains(PReg) && "Not the correct regclass!"); 1013 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1014 MF.getRegInfo().addLiveIn(PReg, VReg); 1015 return VReg; 1016} 1017 1018/// CallIsStructReturn - Determines whether a CALL node uses struct return 1019/// semantics. 1020static bool CallIsStructReturn(SDValue Op) { 1021 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1022 if (!NumOps) 1023 return false; 1024 1025 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1026} 1027 1028/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1029/// return semantics. 1030static bool ArgsAreStructReturn(SDValue Op) { 1031 unsigned NumArgs = Op.Val->getNumValues() - 1; 1032 if (!NumArgs) 1033 return false; 1034 1035 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1036} 1037 1038/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1039/// the callee to pop its own arguments. Callee pop is necessary to support tail 1040/// calls. 1041bool X86TargetLowering::IsCalleePop(SDValue Op) { 1042 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1043 if (IsVarArg) 1044 return false; 1045 1046 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1047 default: 1048 return false; 1049 case CallingConv::X86_StdCall: 1050 return !Subtarget->is64Bit(); 1051 case CallingConv::X86_FastCall: 1052 return !Subtarget->is64Bit(); 1053 case CallingConv::Fast: 1054 return PerformTailCallOpt; 1055 } 1056} 1057 1058/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1059/// FORMAL_ARGUMENTS node. 1060CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDValue Op) const { 1061 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1062 1063 if (Subtarget->is64Bit()) { 1064 if (Subtarget->isTargetWin64()) 1065 return CC_X86_Win64_C; 1066 else { 1067 if (CC == CallingConv::Fast && PerformTailCallOpt) 1068 return CC_X86_64_TailCall; 1069 else 1070 return CC_X86_64_C; 1071 } 1072 } 1073 1074 if (CC == CallingConv::X86_FastCall) 1075 return CC_X86_32_FastCall; 1076 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1077 return CC_X86_32_TailCall; 1078 else 1079 return CC_X86_32_C; 1080} 1081 1082/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1083/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1084NameDecorationStyle 1085X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDValue Op) { 1086 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1087 if (CC == CallingConv::X86_FastCall) 1088 return FastCall; 1089 else if (CC == CallingConv::X86_StdCall) 1090 return StdCall; 1091 return None; 1092} 1093 1094 1095/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1096/// in a register before calling. 1097bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1098 return !IsTailCall && !Is64Bit && 1099 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1100 Subtarget->isPICStyleGOT(); 1101} 1102 1103/// CallRequiresFnAddressInReg - Check whether the call requires the function 1104/// address to be loaded in a register. 1105bool 1106X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1107 return !Is64Bit && IsTailCall && 1108 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1109 Subtarget->isPICStyleGOT(); 1110} 1111 1112/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1113/// by "Src" to address "Dst" with size and alignment information specified by 1114/// the specific parameter attribute. The copy will be passed as a byval 1115/// function parameter. 1116static SDValue 1117CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 1118 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1119 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1120 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1121 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1122} 1123 1124SDValue X86TargetLowering::LowerMemArgument(SDValue Op, SelectionDAG &DAG, 1125 const CCValAssign &VA, 1126 MachineFrameInfo *MFI, 1127 unsigned CC, 1128 SDValue Root, unsigned i) { 1129 // Create the nodes corresponding to a load from this parameter slot. 1130 ISD::ArgFlagsTy Flags = 1131 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1132 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1133 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1134 1135 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1136 // changed with more analysis. 1137 // In case of tail call optimization mark all arguments mutable. Since they 1138 // could be overwritten by lowering of arguments in case of a tail call. 1139 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1140 VA.getLocMemOffset(), isImmutable); 1141 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 1142 if (Flags.isByVal()) 1143 return FIN; 1144 return DAG.getLoad(VA.getValVT(), Root, FIN, 1145 PseudoSourceValue::getFixedStack(FI), 0); 1146} 1147 1148SDValue 1149X86TargetLowering::LowerFORMAL_ARGUMENTS(SDValue Op, SelectionDAG &DAG) { 1150 MachineFunction &MF = DAG.getMachineFunction(); 1151 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1152 1153 const Function* Fn = MF.getFunction(); 1154 if (Fn->hasExternalLinkage() && 1155 Subtarget->isTargetCygMing() && 1156 Fn->getName() == "main") 1157 FuncInfo->setForceFramePointer(true); 1158 1159 // Decorate the function name. 1160 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1161 1162 MachineFrameInfo *MFI = MF.getFrameInfo(); 1163 SDValue Root = Op.getOperand(0); 1164 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1165 unsigned CC = MF.getFunction()->getCallingConv(); 1166 bool Is64Bit = Subtarget->is64Bit(); 1167 bool IsWin64 = Subtarget->isTargetWin64(); 1168 1169 assert(!(isVarArg && CC == CallingConv::Fast) && 1170 "Var args not supported with calling convention fastcc"); 1171 1172 // Assign locations to all of the incoming arguments. 1173 SmallVector<CCValAssign, 16> ArgLocs; 1174 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1175 CCInfo.AnalyzeFormalArguments(Op.Val, CCAssignFnForNode(Op)); 1176 1177 SmallVector<SDValue, 8> ArgValues; 1178 unsigned LastVal = ~0U; 1179 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1180 CCValAssign &VA = ArgLocs[i]; 1181 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1182 // places. 1183 assert(VA.getValNo() != LastVal && 1184 "Don't support value assigned to multiple locs yet"); 1185 LastVal = VA.getValNo(); 1186 1187 if (VA.isRegLoc()) { 1188 MVT RegVT = VA.getLocVT(); 1189 TargetRegisterClass *RC; 1190 if (RegVT == MVT::i32) 1191 RC = X86::GR32RegisterClass; 1192 else if (Is64Bit && RegVT == MVT::i64) 1193 RC = X86::GR64RegisterClass; 1194 else if (RegVT == MVT::f32) 1195 RC = X86::FR32RegisterClass; 1196 else if (RegVT == MVT::f64) 1197 RC = X86::FR64RegisterClass; 1198 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1199 RC = X86::VR128RegisterClass; 1200 else if (RegVT.isVector()) { 1201 assert(RegVT.getSizeInBits() == 64); 1202 if (!Is64Bit) 1203 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1204 else { 1205 // Darwin calling convention passes MMX values in either GPRs or 1206 // XMMs in x86-64. Other targets pass them in memory. 1207 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1208 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1209 RegVT = MVT::v2i64; 1210 } else { 1211 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1212 RegVT = MVT::i64; 1213 } 1214 } 1215 } else { 1216 assert(0 && "Unknown argument type!"); 1217 } 1218 1219 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1220 SDValue ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1221 1222 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1223 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1224 // right size. 1225 if (VA.getLocInfo() == CCValAssign::SExt) 1226 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1227 DAG.getValueType(VA.getValVT())); 1228 else if (VA.getLocInfo() == CCValAssign::ZExt) 1229 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1230 DAG.getValueType(VA.getValVT())); 1231 1232 if (VA.getLocInfo() != CCValAssign::Full) 1233 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1234 1235 // Handle MMX values passed in GPRs. 1236 if (Is64Bit && RegVT != VA.getLocVT()) { 1237 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1238 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1239 else if (RC == X86::VR128RegisterClass) { 1240 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1241 DAG.getConstant(0, MVT::i64)); 1242 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1243 } 1244 } 1245 1246 ArgValues.push_back(ArgValue); 1247 } else { 1248 assert(VA.isMemLoc()); 1249 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1250 } 1251 } 1252 1253 // The x86-64 ABI for returning structs by value requires that we copy 1254 // the sret argument into %rax for the return. Save the argument into 1255 // a virtual register so that we can access it from the return points. 1256 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1257 MachineFunction &MF = DAG.getMachineFunction(); 1258 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1259 unsigned Reg = FuncInfo->getSRetReturnReg(); 1260 if (!Reg) { 1261 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1262 FuncInfo->setSRetReturnReg(Reg); 1263 } 1264 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1265 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1266 } 1267 1268 unsigned StackSize = CCInfo.getNextStackOffset(); 1269 // align stack specially for tail calls 1270 if (CC == CallingConv::Fast) 1271 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1272 1273 // If the function takes variable number of arguments, make a frame index for 1274 // the start of the first vararg value... for expansion of llvm.va_start. 1275 if (isVarArg) { 1276 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1277 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1278 } 1279 if (Is64Bit) { 1280 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1281 1282 // FIXME: We should really autogenerate these arrays 1283 static const unsigned GPR64ArgRegsWin64[] = { 1284 X86::RCX, X86::RDX, X86::R8, X86::R9 1285 }; 1286 static const unsigned XMMArgRegsWin64[] = { 1287 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1288 }; 1289 static const unsigned GPR64ArgRegs64Bit[] = { 1290 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1291 }; 1292 static const unsigned XMMArgRegs64Bit[] = { 1293 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1294 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1295 }; 1296 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1297 1298 if (IsWin64) { 1299 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1300 GPR64ArgRegs = GPR64ArgRegsWin64; 1301 XMMArgRegs = XMMArgRegsWin64; 1302 } else { 1303 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1304 GPR64ArgRegs = GPR64ArgRegs64Bit; 1305 XMMArgRegs = XMMArgRegs64Bit; 1306 } 1307 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1308 TotalNumIntRegs); 1309 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1310 TotalNumXMMRegs); 1311 1312 // For X86-64, if there are vararg parameters that are passed via 1313 // registers, then we must store them to their spots on the stack so they 1314 // may be loaded by deferencing the result of va_next. 1315 VarArgsGPOffset = NumIntRegs * 8; 1316 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1317 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1318 TotalNumXMMRegs * 16, 16); 1319 1320 // Store the integer parameter registers. 1321 SmallVector<SDValue, 8> MemOps; 1322 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1323 SDValue FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1324 DAG.getIntPtrConstant(VarArgsGPOffset)); 1325 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1326 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1327 X86::GR64RegisterClass); 1328 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1329 SDValue Store = 1330 DAG.getStore(Val.getValue(1), Val, FIN, 1331 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1332 MemOps.push_back(Store); 1333 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1334 DAG.getIntPtrConstant(8)); 1335 } 1336 1337 // Now store the XMM (fp + vector) parameter registers. 1338 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1339 DAG.getIntPtrConstant(VarArgsFPOffset)); 1340 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1341 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1342 X86::VR128RegisterClass); 1343 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1344 SDValue Store = 1345 DAG.getStore(Val.getValue(1), Val, FIN, 1346 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1347 MemOps.push_back(Store); 1348 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1349 DAG.getIntPtrConstant(16)); 1350 } 1351 if (!MemOps.empty()) 1352 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1353 &MemOps[0], MemOps.size()); 1354 } 1355 } 1356 1357 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1358 // arguments and the arguments after the retaddr has been pushed are 1359 // aligned. 1360 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1361 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1362 (StackSize & 7) == 0) 1363 StackSize += 4; 1364 1365 ArgValues.push_back(Root); 1366 1367 // Some CCs need callee pop. 1368 if (IsCalleePop(Op)) { 1369 BytesToPopOnReturn = StackSize; // Callee pops everything. 1370 BytesCallerReserves = 0; 1371 } else { 1372 BytesToPopOnReturn = 0; // Callee pops nothing. 1373 // If this is an sret function, the return should pop the hidden pointer. 1374 if (!Is64Bit && ArgsAreStructReturn(Op)) 1375 BytesToPopOnReturn = 4; 1376 BytesCallerReserves = StackSize; 1377 } 1378 1379 if (!Is64Bit) { 1380 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1381 if (CC == CallingConv::X86_FastCall) 1382 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1383 } 1384 1385 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1386 1387 // Return the new list of results. 1388 return DAG.getMergeValues(Op.Val->getVTList(), &ArgValues[0], 1389 ArgValues.size()).getValue(Op.ResNo); 1390} 1391 1392SDValue 1393X86TargetLowering::LowerMemOpCallTo(SDValue Op, SelectionDAG &DAG, 1394 const SDValue &StackPtr, 1395 const CCValAssign &VA, 1396 SDValue Chain, 1397 SDValue Arg) { 1398 unsigned LocMemOffset = VA.getLocMemOffset(); 1399 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1400 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1401 ISD::ArgFlagsTy Flags = 1402 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1403 if (Flags.isByVal()) { 1404 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1405 } 1406 return DAG.getStore(Chain, Arg, PtrOff, 1407 PseudoSourceValue::getStack(), LocMemOffset); 1408} 1409 1410/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1411/// optimization is performed and it is required. 1412SDValue 1413X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1414 SDValue &OutRetAddr, 1415 SDValue Chain, 1416 bool IsTailCall, 1417 bool Is64Bit, 1418 int FPDiff) { 1419 if (!IsTailCall || FPDiff==0) return Chain; 1420 1421 // Adjust the Return address stack slot. 1422 MVT VT = getPointerTy(); 1423 OutRetAddr = getReturnAddressFrameIndex(DAG); 1424 // Load the "old" Return address. 1425 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1426 return SDValue(OutRetAddr.Val, 1); 1427} 1428 1429/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1430/// optimization is performed and it is required (FPDiff!=0). 1431static SDValue 1432EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1433 SDValue Chain, SDValue RetAddrFrIdx, 1434 bool Is64Bit, int FPDiff) { 1435 // Store the return address to the appropriate stack slot. 1436 if (!FPDiff) return Chain; 1437 // Calculate the new stack slot for the return address. 1438 int SlotSize = Is64Bit ? 8 : 4; 1439 int NewReturnAddrFI = 1440 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1441 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1442 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1443 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1444 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 1445 return Chain; 1446} 1447 1448SDValue X86TargetLowering::LowerCALL(SDValue Op, SelectionDAG &DAG) { 1449 MachineFunction &MF = DAG.getMachineFunction(); 1450 SDValue Chain = Op.getOperand(0); 1451 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1452 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1453 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1454 && CC == CallingConv::Fast && PerformTailCallOpt; 1455 SDValue Callee = Op.getOperand(4); 1456 bool Is64Bit = Subtarget->is64Bit(); 1457 bool IsStructRet = CallIsStructReturn(Op); 1458 1459 assert(!(isVarArg && CC == CallingConv::Fast) && 1460 "Var args not supported with calling convention fastcc"); 1461 1462 // Analyze operands of the call, assigning locations to each operand. 1463 SmallVector<CCValAssign, 16> ArgLocs; 1464 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1465 CCInfo.AnalyzeCallOperands(Op.Val, CCAssignFnForNode(Op)); 1466 1467 // Get a count of how many bytes are to be pushed on the stack. 1468 unsigned NumBytes = CCInfo.getNextStackOffset(); 1469 if (CC == CallingConv::Fast) 1470 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1471 1472 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 1473 // arguments and the arguments after the retaddr has been pushed are aligned. 1474 if (!Is64Bit && CC == CallingConv::X86_FastCall && 1475 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWindows() && 1476 (NumBytes & 7) == 0) 1477 NumBytes += 4; 1478 1479 int FPDiff = 0; 1480 if (IsTailCall) { 1481 // Lower arguments at fp - stackoffset + fpdiff. 1482 unsigned NumBytesCallerPushed = 1483 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1484 FPDiff = NumBytesCallerPushed - NumBytes; 1485 1486 // Set the delta of movement of the returnaddr stackslot. 1487 // But only set if delta is greater than previous delta. 1488 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1489 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1490 } 1491 1492 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1493 1494 SDValue RetAddrFrIdx; 1495 // Load return adress for tail calls. 1496 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1497 FPDiff); 1498 1499 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 1500 SmallVector<SDValue, 8> MemOpChains; 1501 SDValue StackPtr; 1502 1503 // Walk the register/memloc assignments, inserting copies/loads. In the case 1504 // of tail call optimization arguments are handle later. 1505 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1506 CCValAssign &VA = ArgLocs[i]; 1507 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1508 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1509 getArgFlags().isByVal(); 1510 1511 // Promote the value if needed. 1512 switch (VA.getLocInfo()) { 1513 default: assert(0 && "Unknown loc info!"); 1514 case CCValAssign::Full: break; 1515 case CCValAssign::SExt: 1516 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1517 break; 1518 case CCValAssign::ZExt: 1519 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1520 break; 1521 case CCValAssign::AExt: 1522 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1523 break; 1524 } 1525 1526 if (VA.isRegLoc()) { 1527 if (Is64Bit) { 1528 MVT RegVT = VA.getLocVT(); 1529 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1530 switch (VA.getLocReg()) { 1531 default: 1532 break; 1533 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1534 case X86::R8: { 1535 // Special case: passing MMX values in GPR registers. 1536 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1537 break; 1538 } 1539 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1540 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1541 // Special case: passing MMX values in XMM registers. 1542 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1543 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1544 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1545 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1546 getMOVLMask(2, DAG)); 1547 break; 1548 } 1549 } 1550 } 1551 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1552 } else { 1553 if (!IsTailCall || (IsTailCall && isByVal)) { 1554 assert(VA.isMemLoc()); 1555 if (StackPtr.Val == 0) 1556 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1557 1558 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1559 Arg)); 1560 } 1561 } 1562 } 1563 1564 if (!MemOpChains.empty()) 1565 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1566 &MemOpChains[0], MemOpChains.size()); 1567 1568 // Build a sequence of copy-to-reg nodes chained together with token chain 1569 // and flag operands which copy the outgoing args into registers. 1570 SDValue InFlag; 1571 // Tail call byval lowering might overwrite argument registers so in case of 1572 // tail call optimization the copies to registers are lowered later. 1573 if (!IsTailCall) 1574 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1575 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1576 InFlag); 1577 InFlag = Chain.getValue(1); 1578 } 1579 1580 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1581 // GOT pointer. 1582 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1583 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1584 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1585 InFlag); 1586 InFlag = Chain.getValue(1); 1587 } 1588 // If we are tail calling and generating PIC/GOT style code load the address 1589 // of the callee into ecx. The value in ecx is used as target of the tail 1590 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1591 // calls on PIC/GOT architectures. Normally we would just put the address of 1592 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1593 // restored (since ebx is callee saved) before jumping to the target@PLT. 1594 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1595 // Note: The actual moving to ecx is done further down. 1596 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1597 if (G && !G->getGlobal()->hasHiddenVisibility() && 1598 !G->getGlobal()->hasProtectedVisibility()) 1599 Callee = LowerGlobalAddress(Callee, DAG); 1600 else if (isa<ExternalSymbolSDNode>(Callee)) 1601 Callee = LowerExternalSymbol(Callee,DAG); 1602 } 1603 1604 if (Is64Bit && isVarArg) { 1605 // From AMD64 ABI document: 1606 // For calls that may call functions that use varargs or stdargs 1607 // (prototype-less calls or calls to functions containing ellipsis (...) in 1608 // the declaration) %al is used as hidden argument to specify the number 1609 // of SSE registers used. The contents of %al do not need to match exactly 1610 // the number of registers, but must be an ubound on the number of SSE 1611 // registers used and is in the range 0 - 8 inclusive. 1612 1613 // FIXME: Verify this on Win64 1614 // Count the number of XMM registers allocated. 1615 static const unsigned XMMArgRegs[] = { 1616 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1617 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1618 }; 1619 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1620 1621 Chain = DAG.getCopyToReg(Chain, X86::AL, 1622 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1623 InFlag = Chain.getValue(1); 1624 } 1625 1626 1627 // For tail calls lower the arguments to the 'real' stack slot. 1628 if (IsTailCall) { 1629 SmallVector<SDValue, 8> MemOpChains2; 1630 SDValue FIN; 1631 int FI = 0; 1632 // Do not flag preceeding copytoreg stuff together with the following stuff. 1633 InFlag = SDValue(); 1634 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1635 CCValAssign &VA = ArgLocs[i]; 1636 if (!VA.isRegLoc()) { 1637 assert(VA.isMemLoc()); 1638 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1639 SDValue FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1640 ISD::ArgFlagsTy Flags = 1641 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1642 // Create frame index. 1643 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1644 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1645 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1646 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1647 1648 if (Flags.isByVal()) { 1649 // Copy relative to framepointer. 1650 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1651 if (StackPtr.Val == 0) 1652 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1653 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1654 1655 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1656 Flags, DAG)); 1657 } else { 1658 // Store relative to framepointer. 1659 MemOpChains2.push_back( 1660 DAG.getStore(Chain, Arg, FIN, 1661 PseudoSourceValue::getFixedStack(FI), 0)); 1662 } 1663 } 1664 } 1665 1666 if (!MemOpChains2.empty()) 1667 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1668 &MemOpChains2[0], MemOpChains2.size()); 1669 1670 // Copy arguments to their registers. 1671 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1672 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1673 InFlag); 1674 InFlag = Chain.getValue(1); 1675 } 1676 InFlag =SDValue(); 1677 1678 // Store the return address to the appropriate stack slot. 1679 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1680 FPDiff); 1681 } 1682 1683 // If the callee is a GlobalAddress node (quite common, every direct call is) 1684 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1685 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1686 // We should use extra load for direct calls to dllimported functions in 1687 // non-JIT mode. 1688 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1689 getTargetMachine(), true)) 1690 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1691 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1692 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1693 } else if (IsTailCall) { 1694 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1695 1696 Chain = DAG.getCopyToReg(Chain, 1697 DAG.getRegister(Opc, getPointerTy()), 1698 Callee,InFlag); 1699 Callee = DAG.getRegister(Opc, getPointerTy()); 1700 // Add register as live out. 1701 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1702 } 1703 1704 // Returns a chain & a flag for retval copy to use. 1705 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1706 SmallVector<SDValue, 8> Ops; 1707 1708 if (IsTailCall) { 1709 Ops.push_back(Chain); 1710 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1711 Ops.push_back(DAG.getIntPtrConstant(0)); 1712 if (InFlag.Val) 1713 Ops.push_back(InFlag); 1714 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1715 InFlag = Chain.getValue(1); 1716 1717 // Returns a chain & a flag for retval copy to use. 1718 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1719 Ops.clear(); 1720 } 1721 1722 Ops.push_back(Chain); 1723 Ops.push_back(Callee); 1724 1725 if (IsTailCall) 1726 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1727 1728 // Add argument registers to the end of the list so that they are known live 1729 // into the call. 1730 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1731 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1732 RegsToPass[i].second.getValueType())); 1733 1734 // Add an implicit use GOT pointer in EBX. 1735 if (!IsTailCall && !Is64Bit && 1736 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1737 Subtarget->isPICStyleGOT()) 1738 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1739 1740 // Add an implicit use of AL for x86 vararg functions. 1741 if (Is64Bit && isVarArg) 1742 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1743 1744 if (InFlag.Val) 1745 Ops.push_back(InFlag); 1746 1747 if (IsTailCall) { 1748 assert(InFlag.Val && 1749 "Flag must be set. Depend on flag being set in LowerRET"); 1750 Chain = DAG.getNode(X86ISD::TAILCALL, 1751 Op.Val->getVTList(), &Ops[0], Ops.size()); 1752 1753 return SDValue(Chain.Val, Op.ResNo); 1754 } 1755 1756 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1757 InFlag = Chain.getValue(1); 1758 1759 // Create the CALLSEQ_END node. 1760 unsigned NumBytesForCalleeToPush; 1761 if (IsCalleePop(Op)) 1762 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1763 else if (!Is64Bit && IsStructRet) 1764 // If this is is a call to a struct-return function, the callee 1765 // pops the hidden struct pointer, so we have to push it back. 1766 // This is common for Darwin/X86, Linux & Mingw32 targets. 1767 NumBytesForCalleeToPush = 4; 1768 else 1769 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1770 1771 // Returns a flag for retval copy to use. 1772 Chain = DAG.getCALLSEQ_END(Chain, 1773 DAG.getIntPtrConstant(NumBytes), 1774 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1775 InFlag); 1776 InFlag = Chain.getValue(1); 1777 1778 // Handle result values, copying them out of physregs into vregs that we 1779 // return. 1780 return SDValue(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo); 1781} 1782 1783 1784//===----------------------------------------------------------------------===// 1785// Fast Calling Convention (tail call) implementation 1786//===----------------------------------------------------------------------===// 1787 1788// Like std call, callee cleans arguments, convention except that ECX is 1789// reserved for storing the tail called function address. Only 2 registers are 1790// free for argument passing (inreg). Tail call optimization is performed 1791// provided: 1792// * tailcallopt is enabled 1793// * caller/callee are fastcc 1794// On X86_64 architecture with GOT-style position independent code only local 1795// (within module) calls are supported at the moment. 1796// To keep the stack aligned according to platform abi the function 1797// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1798// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1799// If a tail called function callee has more arguments than the caller the 1800// caller needs to make sure that there is room to move the RETADDR to. This is 1801// achieved by reserving an area the size of the argument delta right after the 1802// original REtADDR, but before the saved framepointer or the spilled registers 1803// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1804// stack layout: 1805// arg1 1806// arg2 1807// RETADDR 1808// [ new RETADDR 1809// move area ] 1810// (possible EBP) 1811// ESI 1812// EDI 1813// local1 .. 1814 1815/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1816/// for a 16 byte align requirement. 1817unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1818 SelectionDAG& DAG) { 1819 if (PerformTailCallOpt) { 1820 MachineFunction &MF = DAG.getMachineFunction(); 1821 const TargetMachine &TM = MF.getTarget(); 1822 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1823 unsigned StackAlignment = TFI.getStackAlignment(); 1824 uint64_t AlignMask = StackAlignment - 1; 1825 int64_t Offset = StackSize; 1826 unsigned SlotSize = Subtarget->is64Bit() ? 8 : 4; 1827 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1828 // Number smaller than 12 so just add the difference. 1829 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1830 } else { 1831 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1832 Offset = ((~AlignMask) & Offset) + StackAlignment + 1833 (StackAlignment-SlotSize); 1834 } 1835 StackSize = Offset; 1836 } 1837 return StackSize; 1838} 1839 1840/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1841/// following the call is a return. A function is eligible if caller/callee 1842/// calling conventions match, currently only fastcc supports tail calls, and 1843/// the function CALL is immediatly followed by a RET. 1844bool X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Call, 1845 SDValue Ret, 1846 SelectionDAG& DAG) const { 1847 if (!PerformTailCallOpt) 1848 return false; 1849 1850 if (CheckTailCallReturnConstraints(Call, Ret)) { 1851 MachineFunction &MF = DAG.getMachineFunction(); 1852 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1853 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1854 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1855 SDValue Callee = Call.getOperand(4); 1856 // On x86/32Bit PIC/GOT tail calls are supported. 1857 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1858 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1859 return true; 1860 1861 // Can only do local tail calls (in same module, hidden or protected) on 1862 // x86_64 PIC/GOT at the moment. 1863 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1864 return G->getGlobal()->hasHiddenVisibility() 1865 || G->getGlobal()->hasProtectedVisibility(); 1866 } 1867 } 1868 1869 return false; 1870} 1871 1872FastISel *X86TargetLowering::createFastISel(MachineBasicBlock *mbb, 1873 MachineFunction *mf, 1874 const TargetInstrInfo *tii) { 1875 // FastISel isn't yet supported. 1876 return 0; 1877} 1878 1879 1880//===----------------------------------------------------------------------===// 1881// Other Lowering Hooks 1882//===----------------------------------------------------------------------===// 1883 1884 1885SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1886 MachineFunction &MF = DAG.getMachineFunction(); 1887 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1888 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1889 1890 if (ReturnAddrIndex == 0) { 1891 // Set up a frame object for the return address. 1892 if (Subtarget->is64Bit()) 1893 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); 1894 else 1895 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1896 1897 FuncInfo->setRAIndex(ReturnAddrIndex); 1898 } 1899 1900 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1901} 1902 1903 1904 1905/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1906/// specific condition code. It returns a false if it cannot do a direct 1907/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1908/// needed. 1909static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1910 unsigned &X86CC, SDValue &LHS, SDValue &RHS, 1911 SelectionDAG &DAG) { 1912 X86CC = X86::COND_INVALID; 1913 if (!isFP) { 1914 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1915 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1916 // X > -1 -> X == 0, jump !sign. 1917 RHS = DAG.getConstant(0, RHS.getValueType()); 1918 X86CC = X86::COND_NS; 1919 return true; 1920 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1921 // X < 0 -> X == 0, jump on sign. 1922 X86CC = X86::COND_S; 1923 return true; 1924 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1925 // X < 1 -> X <= 0 1926 RHS = DAG.getConstant(0, RHS.getValueType()); 1927 X86CC = X86::COND_LE; 1928 return true; 1929 } 1930 } 1931 1932 switch (SetCCOpcode) { 1933 default: break; 1934 case ISD::SETEQ: X86CC = X86::COND_E; break; 1935 case ISD::SETGT: X86CC = X86::COND_G; break; 1936 case ISD::SETGE: X86CC = X86::COND_GE; break; 1937 case ISD::SETLT: X86CC = X86::COND_L; break; 1938 case ISD::SETLE: X86CC = X86::COND_LE; break; 1939 case ISD::SETNE: X86CC = X86::COND_NE; break; 1940 case ISD::SETULT: X86CC = X86::COND_B; break; 1941 case ISD::SETUGT: X86CC = X86::COND_A; break; 1942 case ISD::SETULE: X86CC = X86::COND_BE; break; 1943 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1944 } 1945 } else { 1946 // On a floating point condition, the flags are set as follows: 1947 // ZF PF CF op 1948 // 0 | 0 | 0 | X > Y 1949 // 0 | 0 | 1 | X < Y 1950 // 1 | 0 | 0 | X == Y 1951 // 1 | 1 | 1 | unordered 1952 bool Flip = false; 1953 switch (SetCCOpcode) { 1954 default: break; 1955 case ISD::SETUEQ: 1956 case ISD::SETEQ: X86CC = X86::COND_E; break; 1957 case ISD::SETOLT: Flip = true; // Fallthrough 1958 case ISD::SETOGT: 1959 case ISD::SETGT: X86CC = X86::COND_A; break; 1960 case ISD::SETOLE: Flip = true; // Fallthrough 1961 case ISD::SETOGE: 1962 case ISD::SETGE: X86CC = X86::COND_AE; break; 1963 case ISD::SETUGT: Flip = true; // Fallthrough 1964 case ISD::SETULT: 1965 case ISD::SETLT: X86CC = X86::COND_B; break; 1966 case ISD::SETUGE: Flip = true; // Fallthrough 1967 case ISD::SETULE: 1968 case ISD::SETLE: X86CC = X86::COND_BE; break; 1969 case ISD::SETONE: 1970 case ISD::SETNE: X86CC = X86::COND_NE; break; 1971 case ISD::SETUO: X86CC = X86::COND_P; break; 1972 case ISD::SETO: X86CC = X86::COND_NP; break; 1973 } 1974 if (Flip) 1975 std::swap(LHS, RHS); 1976 } 1977 1978 return X86CC != X86::COND_INVALID; 1979} 1980 1981/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1982/// code. Current x86 isa includes the following FP cmov instructions: 1983/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1984static bool hasFPCMov(unsigned X86CC) { 1985 switch (X86CC) { 1986 default: 1987 return false; 1988 case X86::COND_B: 1989 case X86::COND_BE: 1990 case X86::COND_E: 1991 case X86::COND_P: 1992 case X86::COND_A: 1993 case X86::COND_AE: 1994 case X86::COND_NE: 1995 case X86::COND_NP: 1996 return true; 1997 } 1998} 1999 2000/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 2001/// true if Op is undef or if its value falls within the specified range (L, H]. 2002static bool isUndefOrInRange(SDValue Op, unsigned Low, unsigned Hi) { 2003 if (Op.getOpcode() == ISD::UNDEF) 2004 return true; 2005 2006 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 2007 return (Val >= Low && Val < Hi); 2008} 2009 2010/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 2011/// true if Op is undef or if its value equal to the specified value. 2012static bool isUndefOrEqual(SDValue Op, unsigned Val) { 2013 if (Op.getOpcode() == ISD::UNDEF) 2014 return true; 2015 return cast<ConstantSDNode>(Op)->getValue() == Val; 2016} 2017 2018/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2019/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2020bool X86::isPSHUFDMask(SDNode *N) { 2021 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2022 2023 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2024 return false; 2025 2026 // Check if the value doesn't reference the second vector. 2027 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2028 SDValue Arg = N->getOperand(i); 2029 if (Arg.getOpcode() == ISD::UNDEF) continue; 2030 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2031 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2032 return false; 2033 } 2034 2035 return true; 2036} 2037 2038/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2039/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2040bool X86::isPSHUFHWMask(SDNode *N) { 2041 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2042 2043 if (N->getNumOperands() != 8) 2044 return false; 2045 2046 // Lower quadword copied in order. 2047 for (unsigned i = 0; i != 4; ++i) { 2048 SDValue Arg = N->getOperand(i); 2049 if (Arg.getOpcode() == ISD::UNDEF) continue; 2050 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2051 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2052 return false; 2053 } 2054 2055 // Upper quadword shuffled. 2056 for (unsigned i = 4; i != 8; ++i) { 2057 SDValue Arg = N->getOperand(i); 2058 if (Arg.getOpcode() == ISD::UNDEF) continue; 2059 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2060 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2061 if (Val < 4 || Val > 7) 2062 return false; 2063 } 2064 2065 return true; 2066} 2067 2068/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2069/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2070bool X86::isPSHUFLWMask(SDNode *N) { 2071 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2072 2073 if (N->getNumOperands() != 8) 2074 return false; 2075 2076 // Upper quadword copied in order. 2077 for (unsigned i = 4; i != 8; ++i) 2078 if (!isUndefOrEqual(N->getOperand(i), i)) 2079 return false; 2080 2081 // Lower quadword shuffled. 2082 for (unsigned i = 0; i != 4; ++i) 2083 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2084 return false; 2085 2086 return true; 2087} 2088 2089/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2090/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2091static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2092 if (NumElems != 2 && NumElems != 4) return false; 2093 2094 unsigned Half = NumElems / 2; 2095 for (unsigned i = 0; i < Half; ++i) 2096 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2097 return false; 2098 for (unsigned i = Half; i < NumElems; ++i) 2099 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2100 return false; 2101 2102 return true; 2103} 2104 2105bool X86::isSHUFPMask(SDNode *N) { 2106 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2107 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2108} 2109 2110/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2111/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2112/// half elements to come from vector 1 (which would equal the dest.) and 2113/// the upper half to come from vector 2. 2114static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2115 if (NumOps != 2 && NumOps != 4) return false; 2116 2117 unsigned Half = NumOps / 2; 2118 for (unsigned i = 0; i < Half; ++i) 2119 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2120 return false; 2121 for (unsigned i = Half; i < NumOps; ++i) 2122 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2123 return false; 2124 return true; 2125} 2126 2127static bool isCommutedSHUFP(SDNode *N) { 2128 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2129 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2130} 2131 2132/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2133/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2134bool X86::isMOVHLPSMask(SDNode *N) { 2135 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2136 2137 if (N->getNumOperands() != 4) 2138 return false; 2139 2140 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2141 return isUndefOrEqual(N->getOperand(0), 6) && 2142 isUndefOrEqual(N->getOperand(1), 7) && 2143 isUndefOrEqual(N->getOperand(2), 2) && 2144 isUndefOrEqual(N->getOperand(3), 3); 2145} 2146 2147/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2148/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2149/// <2, 3, 2, 3> 2150bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2151 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2152 2153 if (N->getNumOperands() != 4) 2154 return false; 2155 2156 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2157 return isUndefOrEqual(N->getOperand(0), 2) && 2158 isUndefOrEqual(N->getOperand(1), 3) && 2159 isUndefOrEqual(N->getOperand(2), 2) && 2160 isUndefOrEqual(N->getOperand(3), 3); 2161} 2162 2163/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2164/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2165bool X86::isMOVLPMask(SDNode *N) { 2166 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2167 2168 unsigned NumElems = N->getNumOperands(); 2169 if (NumElems != 2 && NumElems != 4) 2170 return false; 2171 2172 for (unsigned i = 0; i < NumElems/2; ++i) 2173 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2174 return false; 2175 2176 for (unsigned i = NumElems/2; i < NumElems; ++i) 2177 if (!isUndefOrEqual(N->getOperand(i), i)) 2178 return false; 2179 2180 return true; 2181} 2182 2183/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2184/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2185/// and MOVLHPS. 2186bool X86::isMOVHPMask(SDNode *N) { 2187 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2188 2189 unsigned NumElems = N->getNumOperands(); 2190 if (NumElems != 2 && NumElems != 4) 2191 return false; 2192 2193 for (unsigned i = 0; i < NumElems/2; ++i) 2194 if (!isUndefOrEqual(N->getOperand(i), i)) 2195 return false; 2196 2197 for (unsigned i = 0; i < NumElems/2; ++i) { 2198 SDValue Arg = N->getOperand(i + NumElems/2); 2199 if (!isUndefOrEqual(Arg, i + NumElems)) 2200 return false; 2201 } 2202 2203 return true; 2204} 2205 2206/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2207/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2208bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2209 bool V2IsSplat = false) { 2210 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2211 return false; 2212 2213 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2214 SDValue BitI = Elts[i]; 2215 SDValue BitI1 = Elts[i+1]; 2216 if (!isUndefOrEqual(BitI, j)) 2217 return false; 2218 if (V2IsSplat) { 2219 if (isUndefOrEqual(BitI1, NumElts)) 2220 return false; 2221 } else { 2222 if (!isUndefOrEqual(BitI1, j + NumElts)) 2223 return false; 2224 } 2225 } 2226 2227 return true; 2228} 2229 2230bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2231 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2232 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2233} 2234 2235/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2236/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2237bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2238 bool V2IsSplat = false) { 2239 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2240 return false; 2241 2242 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2243 SDValue BitI = Elts[i]; 2244 SDValue BitI1 = Elts[i+1]; 2245 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2246 return false; 2247 if (V2IsSplat) { 2248 if (isUndefOrEqual(BitI1, NumElts)) 2249 return false; 2250 } else { 2251 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2252 return false; 2253 } 2254 } 2255 2256 return true; 2257} 2258 2259bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2260 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2261 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2262} 2263 2264/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2265/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2266/// <0, 0, 1, 1> 2267bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2268 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2269 2270 unsigned NumElems = N->getNumOperands(); 2271 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2272 return false; 2273 2274 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2275 SDValue BitI = N->getOperand(i); 2276 SDValue BitI1 = N->getOperand(i+1); 2277 2278 if (!isUndefOrEqual(BitI, j)) 2279 return false; 2280 if (!isUndefOrEqual(BitI1, j)) 2281 return false; 2282 } 2283 2284 return true; 2285} 2286 2287/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2288/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2289/// <2, 2, 3, 3> 2290bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2291 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2292 2293 unsigned NumElems = N->getNumOperands(); 2294 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2295 return false; 2296 2297 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2298 SDValue BitI = N->getOperand(i); 2299 SDValue BitI1 = N->getOperand(i + 1); 2300 2301 if (!isUndefOrEqual(BitI, j)) 2302 return false; 2303 if (!isUndefOrEqual(BitI1, j)) 2304 return false; 2305 } 2306 2307 return true; 2308} 2309 2310/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2311/// specifies a shuffle of elements that is suitable for input to MOVSS, 2312/// MOVSD, and MOVD, i.e. setting the lowest element. 2313static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2314 if (NumElts != 2 && NumElts != 4) 2315 return false; 2316 2317 if (!isUndefOrEqual(Elts[0], NumElts)) 2318 return false; 2319 2320 for (unsigned i = 1; i < NumElts; ++i) { 2321 if (!isUndefOrEqual(Elts[i], i)) 2322 return false; 2323 } 2324 2325 return true; 2326} 2327 2328bool X86::isMOVLMask(SDNode *N) { 2329 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2330 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2331} 2332 2333/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2334/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2335/// element of vector 2 and the other elements to come from vector 1 in order. 2336static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2337 bool V2IsSplat = false, 2338 bool V2IsUndef = false) { 2339 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2340 return false; 2341 2342 if (!isUndefOrEqual(Ops[0], 0)) 2343 return false; 2344 2345 for (unsigned i = 1; i < NumOps; ++i) { 2346 SDValue Arg = Ops[i]; 2347 if (!(isUndefOrEqual(Arg, i+NumOps) || 2348 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2349 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2350 return false; 2351 } 2352 2353 return true; 2354} 2355 2356static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2357 bool V2IsUndef = false) { 2358 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2359 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2360 V2IsSplat, V2IsUndef); 2361} 2362 2363/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2364/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2365bool X86::isMOVSHDUPMask(SDNode *N) { 2366 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2367 2368 if (N->getNumOperands() != 4) 2369 return false; 2370 2371 // Expect 1, 1, 3, 3 2372 for (unsigned i = 0; i < 2; ++i) { 2373 SDValue Arg = N->getOperand(i); 2374 if (Arg.getOpcode() == ISD::UNDEF) continue; 2375 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2376 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2377 if (Val != 1) return false; 2378 } 2379 2380 bool HasHi = false; 2381 for (unsigned i = 2; i < 4; ++i) { 2382 SDValue Arg = N->getOperand(i); 2383 if (Arg.getOpcode() == ISD::UNDEF) continue; 2384 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2385 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2386 if (Val != 3) return false; 2387 HasHi = true; 2388 } 2389 2390 // Don't use movshdup if it can be done with a shufps. 2391 return HasHi; 2392} 2393 2394/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2395/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2396bool X86::isMOVSLDUPMask(SDNode *N) { 2397 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2398 2399 if (N->getNumOperands() != 4) 2400 return false; 2401 2402 // Expect 0, 0, 2, 2 2403 for (unsigned i = 0; i < 2; ++i) { 2404 SDValue Arg = N->getOperand(i); 2405 if (Arg.getOpcode() == ISD::UNDEF) continue; 2406 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2407 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2408 if (Val != 0) return false; 2409 } 2410 2411 bool HasHi = false; 2412 for (unsigned i = 2; i < 4; ++i) { 2413 SDValue Arg = N->getOperand(i); 2414 if (Arg.getOpcode() == ISD::UNDEF) continue; 2415 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2416 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2417 if (Val != 2) return false; 2418 HasHi = true; 2419 } 2420 2421 // Don't use movshdup if it can be done with a shufps. 2422 return HasHi; 2423} 2424 2425/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2426/// specifies a identity operation on the LHS or RHS. 2427static bool isIdentityMask(SDNode *N, bool RHS = false) { 2428 unsigned NumElems = N->getNumOperands(); 2429 for (unsigned i = 0; i < NumElems; ++i) 2430 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2431 return false; 2432 return true; 2433} 2434 2435/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2436/// a splat of a single element. 2437static bool isSplatMask(SDNode *N) { 2438 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2439 2440 // This is a splat operation if each element of the permute is the same, and 2441 // if the value doesn't reference the second vector. 2442 unsigned NumElems = N->getNumOperands(); 2443 SDValue ElementBase; 2444 unsigned i = 0; 2445 for (; i != NumElems; ++i) { 2446 SDValue Elt = N->getOperand(i); 2447 if (isa<ConstantSDNode>(Elt)) { 2448 ElementBase = Elt; 2449 break; 2450 } 2451 } 2452 2453 if (!ElementBase.Val) 2454 return false; 2455 2456 for (; i != NumElems; ++i) { 2457 SDValue Arg = N->getOperand(i); 2458 if (Arg.getOpcode() == ISD::UNDEF) continue; 2459 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2460 if (Arg != ElementBase) return false; 2461 } 2462 2463 // Make sure it is a splat of the first vector operand. 2464 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2465} 2466 2467/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2468/// a splat of a single element and it's a 2 or 4 element mask. 2469bool X86::isSplatMask(SDNode *N) { 2470 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2471 2472 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2473 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2474 return false; 2475 return ::isSplatMask(N); 2476} 2477 2478/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2479/// specifies a splat of zero element. 2480bool X86::isSplatLoMask(SDNode *N) { 2481 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2482 2483 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2484 if (!isUndefOrEqual(N->getOperand(i), 0)) 2485 return false; 2486 return true; 2487} 2488 2489/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2490/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2491/// instructions. 2492unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2493 unsigned NumOperands = N->getNumOperands(); 2494 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2495 unsigned Mask = 0; 2496 for (unsigned i = 0; i < NumOperands; ++i) { 2497 unsigned Val = 0; 2498 SDValue Arg = N->getOperand(NumOperands-i-1); 2499 if (Arg.getOpcode() != ISD::UNDEF) 2500 Val = cast<ConstantSDNode>(Arg)->getValue(); 2501 if (Val >= NumOperands) Val -= NumOperands; 2502 Mask |= Val; 2503 if (i != NumOperands - 1) 2504 Mask <<= Shift; 2505 } 2506 2507 return Mask; 2508} 2509 2510/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2511/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2512/// instructions. 2513unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2514 unsigned Mask = 0; 2515 // 8 nodes, but we only care about the last 4. 2516 for (unsigned i = 7; i >= 4; --i) { 2517 unsigned Val = 0; 2518 SDValue Arg = N->getOperand(i); 2519 if (Arg.getOpcode() != ISD::UNDEF) 2520 Val = cast<ConstantSDNode>(Arg)->getValue(); 2521 Mask |= (Val - 4); 2522 if (i != 4) 2523 Mask <<= 2; 2524 } 2525 2526 return Mask; 2527} 2528 2529/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2530/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2531/// instructions. 2532unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2533 unsigned Mask = 0; 2534 // 8 nodes, but we only care about the first 4. 2535 for (int i = 3; i >= 0; --i) { 2536 unsigned Val = 0; 2537 SDValue Arg = N->getOperand(i); 2538 if (Arg.getOpcode() != ISD::UNDEF) 2539 Val = cast<ConstantSDNode>(Arg)->getValue(); 2540 Mask |= Val; 2541 if (i != 0) 2542 Mask <<= 2; 2543 } 2544 2545 return Mask; 2546} 2547 2548/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2549/// specifies a 8 element shuffle that can be broken into a pair of 2550/// PSHUFHW and PSHUFLW. 2551static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2552 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2553 2554 if (N->getNumOperands() != 8) 2555 return false; 2556 2557 // Lower quadword shuffled. 2558 for (unsigned i = 0; i != 4; ++i) { 2559 SDValue Arg = N->getOperand(i); 2560 if (Arg.getOpcode() == ISD::UNDEF) continue; 2561 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2562 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2563 if (Val >= 4) 2564 return false; 2565 } 2566 2567 // Upper quadword shuffled. 2568 for (unsigned i = 4; i != 8; ++i) { 2569 SDValue Arg = N->getOperand(i); 2570 if (Arg.getOpcode() == ISD::UNDEF) continue; 2571 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2572 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2573 if (Val < 4 || Val > 7) 2574 return false; 2575 } 2576 2577 return true; 2578} 2579 2580/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2581/// values in ther permute mask. 2582static SDValue CommuteVectorShuffle(SDValue Op, SDValue &V1, 2583 SDValue &V2, SDValue &Mask, 2584 SelectionDAG &DAG) { 2585 MVT VT = Op.getValueType(); 2586 MVT MaskVT = Mask.getValueType(); 2587 MVT EltVT = MaskVT.getVectorElementType(); 2588 unsigned NumElems = Mask.getNumOperands(); 2589 SmallVector<SDValue, 8> MaskVec; 2590 2591 for (unsigned i = 0; i != NumElems; ++i) { 2592 SDValue Arg = Mask.getOperand(i); 2593 if (Arg.getOpcode() == ISD::UNDEF) { 2594 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2595 continue; 2596 } 2597 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2598 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2599 if (Val < NumElems) 2600 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2601 else 2602 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2603 } 2604 2605 std::swap(V1, V2); 2606 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2607 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2608} 2609 2610/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2611/// the two vector operands have swapped position. 2612static 2613SDValue CommuteVectorShuffleMask(SDValue Mask, SelectionDAG &DAG) { 2614 MVT MaskVT = Mask.getValueType(); 2615 MVT EltVT = MaskVT.getVectorElementType(); 2616 unsigned NumElems = Mask.getNumOperands(); 2617 SmallVector<SDValue, 8> MaskVec; 2618 for (unsigned i = 0; i != NumElems; ++i) { 2619 SDValue Arg = Mask.getOperand(i); 2620 if (Arg.getOpcode() == ISD::UNDEF) { 2621 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2622 continue; 2623 } 2624 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2625 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2626 if (Val < NumElems) 2627 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2628 else 2629 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2630 } 2631 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2632} 2633 2634 2635/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2636/// match movhlps. The lower half elements should come from upper half of 2637/// V1 (and in order), and the upper half elements should come from the upper 2638/// half of V2 (and in order). 2639static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2640 unsigned NumElems = Mask->getNumOperands(); 2641 if (NumElems != 4) 2642 return false; 2643 for (unsigned i = 0, e = 2; i != e; ++i) 2644 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2645 return false; 2646 for (unsigned i = 2; i != 4; ++i) 2647 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2648 return false; 2649 return true; 2650} 2651 2652/// isScalarLoadToVector - Returns true if the node is a scalar load that 2653/// is promoted to a vector. It also returns the LoadSDNode by reference if 2654/// required. 2655static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2656 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2657 N = N->getOperand(0).Val; 2658 if (ISD::isNON_EXTLoad(N)) { 2659 if (LD) 2660 *LD = cast<LoadSDNode>(N); 2661 return true; 2662 } 2663 } 2664 return false; 2665} 2666 2667/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2668/// match movlp{s|d}. The lower half elements should come from lower half of 2669/// V1 (and in order), and the upper half elements should come from the upper 2670/// half of V2 (and in order). And since V1 will become the source of the 2671/// MOVLP, it must be either a vector load or a scalar load to vector. 2672static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2673 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2674 return false; 2675 // Is V2 is a vector load, don't do this transformation. We will try to use 2676 // load folding shufps op. 2677 if (ISD::isNON_EXTLoad(V2)) 2678 return false; 2679 2680 unsigned NumElems = Mask->getNumOperands(); 2681 if (NumElems != 2 && NumElems != 4) 2682 return false; 2683 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2684 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2685 return false; 2686 for (unsigned i = NumElems/2; i != NumElems; ++i) 2687 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2688 return false; 2689 return true; 2690} 2691 2692/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2693/// all the same. 2694static bool isSplatVector(SDNode *N) { 2695 if (N->getOpcode() != ISD::BUILD_VECTOR) 2696 return false; 2697 2698 SDValue SplatValue = N->getOperand(0); 2699 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2700 if (N->getOperand(i) != SplatValue) 2701 return false; 2702 return true; 2703} 2704 2705/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2706/// to an undef. 2707static bool isUndefShuffle(SDNode *N) { 2708 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2709 return false; 2710 2711 SDValue V1 = N->getOperand(0); 2712 SDValue V2 = N->getOperand(1); 2713 SDValue Mask = N->getOperand(2); 2714 unsigned NumElems = Mask.getNumOperands(); 2715 for (unsigned i = 0; i != NumElems; ++i) { 2716 SDValue Arg = Mask.getOperand(i); 2717 if (Arg.getOpcode() != ISD::UNDEF) { 2718 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2719 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2720 return false; 2721 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2722 return false; 2723 } 2724 } 2725 return true; 2726} 2727 2728/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2729/// constant +0.0. 2730static inline bool isZeroNode(SDValue Elt) { 2731 return ((isa<ConstantSDNode>(Elt) && 2732 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2733 (isa<ConstantFPSDNode>(Elt) && 2734 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2735} 2736 2737/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2738/// to an zero vector. 2739static bool isZeroShuffle(SDNode *N) { 2740 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2741 return false; 2742 2743 SDValue V1 = N->getOperand(0); 2744 SDValue V2 = N->getOperand(1); 2745 SDValue Mask = N->getOperand(2); 2746 unsigned NumElems = Mask.getNumOperands(); 2747 for (unsigned i = 0; i != NumElems; ++i) { 2748 SDValue Arg = Mask.getOperand(i); 2749 if (Arg.getOpcode() == ISD::UNDEF) 2750 continue; 2751 2752 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2753 if (Idx < NumElems) { 2754 unsigned Opc = V1.Val->getOpcode(); 2755 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.Val)) 2756 continue; 2757 if (Opc != ISD::BUILD_VECTOR || 2758 !isZeroNode(V1.Val->getOperand(Idx))) 2759 return false; 2760 } else if (Idx >= NumElems) { 2761 unsigned Opc = V2.Val->getOpcode(); 2762 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.Val)) 2763 continue; 2764 if (Opc != ISD::BUILD_VECTOR || 2765 !isZeroNode(V2.Val->getOperand(Idx - NumElems))) 2766 return false; 2767 } 2768 } 2769 return true; 2770} 2771 2772/// getZeroVector - Returns a vector of specified type with all zero elements. 2773/// 2774static SDValue getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2775 assert(VT.isVector() && "Expected a vector type"); 2776 2777 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2778 // type. This ensures they get CSE'd. 2779 SDValue Vec; 2780 if (VT.getSizeInBits() == 64) { // MMX 2781 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2782 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2783 } else if (HasSSE2) { // SSE2 2784 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2785 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2786 } else { // SSE1 2787 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2788 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2789 } 2790 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2791} 2792 2793/// getOnesVector - Returns a vector of specified type with all bits set. 2794/// 2795static SDValue getOnesVector(MVT VT, SelectionDAG &DAG) { 2796 assert(VT.isVector() && "Expected a vector type"); 2797 2798 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2799 // type. This ensures they get CSE'd. 2800 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 2801 SDValue Vec; 2802 if (VT.getSizeInBits() == 64) // MMX 2803 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2804 else // SSE 2805 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2806 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2807} 2808 2809 2810/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2811/// that point to V2 points to its first element. 2812static SDValue NormalizeMask(SDValue Mask, SelectionDAG &DAG) { 2813 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2814 2815 bool Changed = false; 2816 SmallVector<SDValue, 8> MaskVec; 2817 unsigned NumElems = Mask.getNumOperands(); 2818 for (unsigned i = 0; i != NumElems; ++i) { 2819 SDValue Arg = Mask.getOperand(i); 2820 if (Arg.getOpcode() != ISD::UNDEF) { 2821 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2822 if (Val > NumElems) { 2823 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2824 Changed = true; 2825 } 2826 } 2827 MaskVec.push_back(Arg); 2828 } 2829 2830 if (Changed) 2831 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2832 &MaskVec[0], MaskVec.size()); 2833 return Mask; 2834} 2835 2836/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2837/// operation of specified width. 2838static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2839 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2840 MVT BaseVT = MaskVT.getVectorElementType(); 2841 2842 SmallVector<SDValue, 8> MaskVec; 2843 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2844 for (unsigned i = 1; i != NumElems; ++i) 2845 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2846 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2847} 2848 2849/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2850/// of specified width. 2851static SDValue getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2852 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2853 MVT BaseVT = MaskVT.getVectorElementType(); 2854 SmallVector<SDValue, 8> MaskVec; 2855 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2856 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2857 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2858 } 2859 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2860} 2861 2862/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2863/// of specified width. 2864static SDValue getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2865 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2866 MVT BaseVT = MaskVT.getVectorElementType(); 2867 unsigned Half = NumElems/2; 2868 SmallVector<SDValue, 8> MaskVec; 2869 for (unsigned i = 0; i != Half; ++i) { 2870 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2871 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2872 } 2873 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2874} 2875 2876/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2877/// element #0 of a vector with the specified index, leaving the rest of the 2878/// elements in place. 2879static SDValue getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2880 SelectionDAG &DAG) { 2881 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2882 MVT BaseVT = MaskVT.getVectorElementType(); 2883 SmallVector<SDValue, 8> MaskVec; 2884 // Element #0 of the result gets the elt we are replacing. 2885 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2886 for (unsigned i = 1; i != NumElems; ++i) 2887 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2888 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2889} 2890 2891/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2892static SDValue PromoteSplat(SDValue Op, SelectionDAG &DAG, bool HasSSE2) { 2893 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2894 MVT VT = Op.getValueType(); 2895 if (PVT == VT) 2896 return Op; 2897 SDValue V1 = Op.getOperand(0); 2898 SDValue Mask = Op.getOperand(2); 2899 unsigned NumElems = Mask.getNumOperands(); 2900 // Special handling of v4f32 -> v4i32. 2901 if (VT != MVT::v4f32) { 2902 Mask = getUnpacklMask(NumElems, DAG); 2903 while (NumElems > 4) { 2904 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2905 NumElems >>= 1; 2906 } 2907 Mask = getZeroVector(MVT::v4i32, true, DAG); 2908 } 2909 2910 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2911 SDValue Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2912 DAG.getNode(ISD::UNDEF, PVT), Mask); 2913 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2914} 2915 2916/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2917/// vector of zero or undef vector. This produces a shuffle where the low 2918/// element of V2 is swizzled into the zero/undef vector, landing at element 2919/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2920static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 2921 bool isZero, bool HasSSE2, 2922 SelectionDAG &DAG) { 2923 MVT VT = V2.getValueType(); 2924 SDValue V1 = isZero 2925 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2926 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2927 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2928 MVT EVT = MaskVT.getVectorElementType(); 2929 SmallVector<SDValue, 16> MaskVec; 2930 for (unsigned i = 0; i != NumElems; ++i) 2931 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2932 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2933 else 2934 MaskVec.push_back(DAG.getConstant(i, EVT)); 2935 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2936 &MaskVec[0], MaskVec.size()); 2937 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2938} 2939 2940/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2941/// a shuffle that is zero. 2942static 2943unsigned getNumOfConsecutiveZeros(SDValue Op, SDValue Mask, 2944 unsigned NumElems, bool Low, 2945 SelectionDAG &DAG) { 2946 unsigned NumZeros = 0; 2947 for (unsigned i = 0; i < NumElems; ++i) { 2948 unsigned Index = Low ? i : NumElems-i-1; 2949 SDValue Idx = Mask.getOperand(Index); 2950 if (Idx.getOpcode() == ISD::UNDEF) { 2951 ++NumZeros; 2952 continue; 2953 } 2954 SDValue Elt = DAG.getShuffleScalarElt(Op.Val, Index); 2955 if (Elt.Val && isZeroNode(Elt)) 2956 ++NumZeros; 2957 else 2958 break; 2959 } 2960 return NumZeros; 2961} 2962 2963/// isVectorShift - Returns true if the shuffle can be implemented as a 2964/// logical left or right shift of a vector. 2965static bool isVectorShift(SDValue Op, SDValue Mask, SelectionDAG &DAG, 2966 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 2967 unsigned NumElems = Mask.getNumOperands(); 2968 2969 isLeft = true; 2970 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 2971 if (!NumZeros) { 2972 isLeft = false; 2973 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 2974 if (!NumZeros) 2975 return false; 2976 } 2977 2978 bool SeenV1 = false; 2979 bool SeenV2 = false; 2980 for (unsigned i = NumZeros; i < NumElems; ++i) { 2981 unsigned Val = isLeft ? (i - NumZeros) : i; 2982 SDValue Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 2983 if (Idx.getOpcode() == ISD::UNDEF) 2984 continue; 2985 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 2986 if (Index < NumElems) 2987 SeenV1 = true; 2988 else { 2989 Index -= NumElems; 2990 SeenV2 = true; 2991 } 2992 if (Index != Val) 2993 return false; 2994 } 2995 if (SeenV1 && SeenV2) 2996 return false; 2997 2998 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 2999 ShAmt = NumZeros; 3000 return true; 3001} 3002 3003 3004/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 3005/// 3006static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 3007 unsigned NumNonZero, unsigned NumZero, 3008 SelectionDAG &DAG, TargetLowering &TLI) { 3009 if (NumNonZero > 8) 3010 return SDValue(); 3011 3012 SDValue V(0, 0); 3013 bool First = true; 3014 for (unsigned i = 0; i < 16; ++i) { 3015 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3016 if (ThisIsNonZero && First) { 3017 if (NumZero) 3018 V = getZeroVector(MVT::v8i16, true, DAG); 3019 else 3020 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3021 First = false; 3022 } 3023 3024 if ((i & 1) != 0) { 3025 SDValue ThisElt(0, 0), LastElt(0, 0); 3026 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3027 if (LastIsNonZero) { 3028 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3029 } 3030 if (ThisIsNonZero) { 3031 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3032 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3033 ThisElt, DAG.getConstant(8, MVT::i8)); 3034 if (LastIsNonZero) 3035 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3036 } else 3037 ThisElt = LastElt; 3038 3039 if (ThisElt.Val) 3040 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3041 DAG.getIntPtrConstant(i/2)); 3042 } 3043 } 3044 3045 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3046} 3047 3048/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3049/// 3050static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 3051 unsigned NumNonZero, unsigned NumZero, 3052 SelectionDAG &DAG, TargetLowering &TLI) { 3053 if (NumNonZero > 4) 3054 return SDValue(); 3055 3056 SDValue V(0, 0); 3057 bool First = true; 3058 for (unsigned i = 0; i < 8; ++i) { 3059 bool isNonZero = (NonZeros & (1 << i)) != 0; 3060 if (isNonZero) { 3061 if (First) { 3062 if (NumZero) 3063 V = getZeroVector(MVT::v8i16, true, DAG); 3064 else 3065 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3066 First = false; 3067 } 3068 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3069 DAG.getIntPtrConstant(i)); 3070 } 3071 } 3072 3073 return V; 3074} 3075 3076/// getVShift - Return a vector logical shift node. 3077/// 3078static SDValue getVShift(bool isLeft, MVT VT, SDValue SrcOp, 3079 unsigned NumBits, SelectionDAG &DAG, 3080 const TargetLowering &TLI) { 3081 bool isMMX = VT.getSizeInBits() == 64; 3082 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3083 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3084 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3085 return DAG.getNode(ISD::BIT_CONVERT, VT, 3086 DAG.getNode(Opc, ShVT, SrcOp, 3087 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3088} 3089 3090SDValue 3091X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) { 3092 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3093 if (ISD::isBuildVectorAllZeros(Op.Val) || ISD::isBuildVectorAllOnes(Op.Val)) { 3094 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3095 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3096 // eliminated on x86-32 hosts. 3097 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3098 return Op; 3099 3100 if (ISD::isBuildVectorAllOnes(Op.Val)) 3101 return getOnesVector(Op.getValueType(), DAG); 3102 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3103 } 3104 3105 MVT VT = Op.getValueType(); 3106 MVT EVT = VT.getVectorElementType(); 3107 unsigned EVTBits = EVT.getSizeInBits(); 3108 3109 unsigned NumElems = Op.getNumOperands(); 3110 unsigned NumZero = 0; 3111 unsigned NumNonZero = 0; 3112 unsigned NonZeros = 0; 3113 bool IsAllConstants = true; 3114 SmallSet<SDValue, 8> Values; 3115 for (unsigned i = 0; i < NumElems; ++i) { 3116 SDValue Elt = Op.getOperand(i); 3117 if (Elt.getOpcode() == ISD::UNDEF) 3118 continue; 3119 Values.insert(Elt); 3120 if (Elt.getOpcode() != ISD::Constant && 3121 Elt.getOpcode() != ISD::ConstantFP) 3122 IsAllConstants = false; 3123 if (isZeroNode(Elt)) 3124 NumZero++; 3125 else { 3126 NonZeros |= (1 << i); 3127 NumNonZero++; 3128 } 3129 } 3130 3131 if (NumNonZero == 0) { 3132 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3133 return DAG.getNode(ISD::UNDEF, VT); 3134 } 3135 3136 // Special case for single non-zero, non-undef, element. 3137 if (NumNonZero == 1 && NumElems <= 4) { 3138 unsigned Idx = CountTrailingZeros_32(NonZeros); 3139 SDValue Item = Op.getOperand(Idx); 3140 3141 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3142 // the value are obviously zero, truncate the value to i32 and do the 3143 // insertion that way. Only do this if the value is non-constant or if the 3144 // value is a constant being inserted into element 0. It is cheaper to do 3145 // a constant pool load than it is to do a movd + shuffle. 3146 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3147 (!IsAllConstants || Idx == 0)) { 3148 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3149 // Handle MMX and SSE both. 3150 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3151 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3152 3153 // Truncate the value (which may itself be a constant) to i32, and 3154 // convert it to a vector with movd (S2V+shuffle to zero extend). 3155 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3156 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3157 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3158 Subtarget->hasSSE2(), DAG); 3159 3160 // Now we have our 32-bit value zero extended in the low element of 3161 // a vector. If Idx != 0, swizzle it into place. 3162 if (Idx != 0) { 3163 SDValue Ops[] = { 3164 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3165 getSwapEltZeroMask(VecElts, Idx, DAG) 3166 }; 3167 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3168 } 3169 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3170 } 3171 } 3172 3173 // If we have a constant or non-constant insertion into the low element of 3174 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3175 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3176 // depending on what the source datatype is. Because we can only get here 3177 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3178 if (Idx == 0 && 3179 // Don't do this for i64 values on x86-32. 3180 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3181 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3182 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3183 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3184 Subtarget->hasSSE2(), DAG); 3185 } 3186 3187 // Is it a vector logical left shift? 3188 if (NumElems == 2 && Idx == 1 && 3189 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3190 unsigned NumBits = VT.getSizeInBits(); 3191 return getVShift(true, VT, 3192 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3193 NumBits/2, DAG, *this); 3194 } 3195 3196 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3197 return SDValue(); 3198 3199 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3200 // is a non-constant being inserted into an element other than the low one, 3201 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3202 // movd/movss) to move this into the low element, then shuffle it into 3203 // place. 3204 if (EVTBits == 32) { 3205 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3206 3207 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3208 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3209 Subtarget->hasSSE2(), DAG); 3210 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3211 MVT MaskEVT = MaskVT.getVectorElementType(); 3212 SmallVector<SDValue, 8> MaskVec; 3213 for (unsigned i = 0; i < NumElems; i++) 3214 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3215 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3216 &MaskVec[0], MaskVec.size()); 3217 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3218 DAG.getNode(ISD::UNDEF, VT), Mask); 3219 } 3220 } 3221 3222 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3223 if (Values.size() == 1) 3224 return SDValue(); 3225 3226 // A vector full of immediates; various special cases are already 3227 // handled, so this is best done with a single constant-pool load. 3228 if (IsAllConstants) 3229 return SDValue(); 3230 3231 // Let legalizer expand 2-wide build_vectors. 3232 if (EVTBits == 64) { 3233 if (NumNonZero == 1) { 3234 // One half is zero or undef. 3235 unsigned Idx = CountTrailingZeros_32(NonZeros); 3236 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3237 Op.getOperand(Idx)); 3238 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3239 Subtarget->hasSSE2(), DAG); 3240 } 3241 return SDValue(); 3242 } 3243 3244 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3245 if (EVTBits == 8 && NumElems == 16) { 3246 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3247 *this); 3248 if (V.Val) return V; 3249 } 3250 3251 if (EVTBits == 16 && NumElems == 8) { 3252 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3253 *this); 3254 if (V.Val) return V; 3255 } 3256 3257 // If element VT is == 32 bits, turn it into a number of shuffles. 3258 SmallVector<SDValue, 8> V; 3259 V.resize(NumElems); 3260 if (NumElems == 4 && NumZero > 0) { 3261 for (unsigned i = 0; i < 4; ++i) { 3262 bool isZero = !(NonZeros & (1 << i)); 3263 if (isZero) 3264 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3265 else 3266 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3267 } 3268 3269 for (unsigned i = 0; i < 2; ++i) { 3270 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3271 default: break; 3272 case 0: 3273 V[i] = V[i*2]; // Must be a zero vector. 3274 break; 3275 case 1: 3276 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3277 getMOVLMask(NumElems, DAG)); 3278 break; 3279 case 2: 3280 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3281 getMOVLMask(NumElems, DAG)); 3282 break; 3283 case 3: 3284 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3285 getUnpacklMask(NumElems, DAG)); 3286 break; 3287 } 3288 } 3289 3290 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3291 MVT EVT = MaskVT.getVectorElementType(); 3292 SmallVector<SDValue, 8> MaskVec; 3293 bool Reverse = (NonZeros & 0x3) == 2; 3294 for (unsigned i = 0; i < 2; ++i) 3295 if (Reverse) 3296 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3297 else 3298 MaskVec.push_back(DAG.getConstant(i, EVT)); 3299 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3300 for (unsigned i = 0; i < 2; ++i) 3301 if (Reverse) 3302 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3303 else 3304 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3305 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3306 &MaskVec[0], MaskVec.size()); 3307 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3308 } 3309 3310 if (Values.size() > 2) { 3311 // Expand into a number of unpckl*. 3312 // e.g. for v4f32 3313 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3314 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3315 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3316 SDValue UnpckMask = getUnpacklMask(NumElems, DAG); 3317 for (unsigned i = 0; i < NumElems; ++i) 3318 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3319 NumElems >>= 1; 3320 while (NumElems != 0) { 3321 for (unsigned i = 0; i < NumElems; ++i) 3322 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3323 UnpckMask); 3324 NumElems >>= 1; 3325 } 3326 return V[0]; 3327 } 3328 3329 return SDValue(); 3330} 3331 3332static 3333SDValue LowerVECTOR_SHUFFLEv8i16(SDValue V1, SDValue V2, 3334 SDValue PermMask, SelectionDAG &DAG, 3335 TargetLowering &TLI) { 3336 SDValue NewV; 3337 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3338 MVT MaskEVT = MaskVT.getVectorElementType(); 3339 MVT PtrVT = TLI.getPointerTy(); 3340 SmallVector<SDValue, 8> MaskElts(PermMask.Val->op_begin(), 3341 PermMask.Val->op_end()); 3342 3343 // First record which half of which vector the low elements come from. 3344 SmallVector<unsigned, 4> LowQuad(4); 3345 for (unsigned i = 0; i < 4; ++i) { 3346 SDValue Elt = MaskElts[i]; 3347 if (Elt.getOpcode() == ISD::UNDEF) 3348 continue; 3349 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3350 int QuadIdx = EltIdx / 4; 3351 ++LowQuad[QuadIdx]; 3352 } 3353 int BestLowQuad = -1; 3354 unsigned MaxQuad = 1; 3355 for (unsigned i = 0; i < 4; ++i) { 3356 if (LowQuad[i] > MaxQuad) { 3357 BestLowQuad = i; 3358 MaxQuad = LowQuad[i]; 3359 } 3360 } 3361 3362 // Record which half of which vector the high elements come from. 3363 SmallVector<unsigned, 4> HighQuad(4); 3364 for (unsigned i = 4; i < 8; ++i) { 3365 SDValue Elt = MaskElts[i]; 3366 if (Elt.getOpcode() == ISD::UNDEF) 3367 continue; 3368 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3369 int QuadIdx = EltIdx / 4; 3370 ++HighQuad[QuadIdx]; 3371 } 3372 int BestHighQuad = -1; 3373 MaxQuad = 1; 3374 for (unsigned i = 0; i < 4; ++i) { 3375 if (HighQuad[i] > MaxQuad) { 3376 BestHighQuad = i; 3377 MaxQuad = HighQuad[i]; 3378 } 3379 } 3380 3381 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3382 if (BestLowQuad != -1 || BestHighQuad != -1) { 3383 // First sort the 4 chunks in order using shufpd. 3384 SmallVector<SDValue, 8> MaskVec; 3385 if (BestLowQuad != -1) 3386 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3387 else 3388 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3389 if (BestHighQuad != -1) 3390 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3391 else 3392 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3393 SDValue Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3394 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3395 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3396 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3397 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3398 3399 // Now sort high and low parts separately. 3400 BitVector InOrder(8); 3401 if (BestLowQuad != -1) { 3402 // Sort lower half in order using PSHUFLW. 3403 MaskVec.clear(); 3404 bool AnyOutOrder = false; 3405 for (unsigned i = 0; i != 4; ++i) { 3406 SDValue Elt = MaskElts[i]; 3407 if (Elt.getOpcode() == ISD::UNDEF) { 3408 MaskVec.push_back(Elt); 3409 InOrder.set(i); 3410 } else { 3411 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3412 if (EltIdx != i) 3413 AnyOutOrder = true; 3414 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3415 // If this element is in the right place after this shuffle, then 3416 // remember it. 3417 if ((int)(EltIdx / 4) == BestLowQuad) 3418 InOrder.set(i); 3419 } 3420 } 3421 if (AnyOutOrder) { 3422 for (unsigned i = 4; i != 8; ++i) 3423 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3424 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3425 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3426 } 3427 } 3428 3429 if (BestHighQuad != -1) { 3430 // Sort high half in order using PSHUFHW if possible. 3431 MaskVec.clear(); 3432 for (unsigned i = 0; i != 4; ++i) 3433 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3434 bool AnyOutOrder = false; 3435 for (unsigned i = 4; i != 8; ++i) { 3436 SDValue Elt = MaskElts[i]; 3437 if (Elt.getOpcode() == ISD::UNDEF) { 3438 MaskVec.push_back(Elt); 3439 InOrder.set(i); 3440 } else { 3441 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3442 if (EltIdx != i) 3443 AnyOutOrder = true; 3444 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3445 // If this element is in the right place after this shuffle, then 3446 // remember it. 3447 if ((int)(EltIdx / 4) == BestHighQuad) 3448 InOrder.set(i); 3449 } 3450 } 3451 if (AnyOutOrder) { 3452 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3453 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3454 } 3455 } 3456 3457 // The other elements are put in the right place using pextrw and pinsrw. 3458 for (unsigned i = 0; i != 8; ++i) { 3459 if (InOrder[i]) 3460 continue; 3461 SDValue Elt = MaskElts[i]; 3462 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3463 SDValue ExtOp = (EltIdx < 8) 3464 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3465 DAG.getConstant(EltIdx, PtrVT)) 3466 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3467 DAG.getConstant(EltIdx - 8, PtrVT)); 3468 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3469 DAG.getConstant(i, PtrVT)); 3470 } 3471 return NewV; 3472 } 3473 3474 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use 3475 ///as few as possible. 3476 // First, let's find out how many elements are already in the right order. 3477 unsigned V1InOrder = 0; 3478 unsigned V1FromV1 = 0; 3479 unsigned V2InOrder = 0; 3480 unsigned V2FromV2 = 0; 3481 SmallVector<SDValue, 8> V1Elts; 3482 SmallVector<SDValue, 8> V2Elts; 3483 for (unsigned i = 0; i < 8; ++i) { 3484 SDValue Elt = MaskElts[i]; 3485 if (Elt.getOpcode() == ISD::UNDEF) { 3486 V1Elts.push_back(Elt); 3487 V2Elts.push_back(Elt); 3488 ++V1InOrder; 3489 ++V2InOrder; 3490 continue; 3491 } 3492 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3493 if (EltIdx == i) { 3494 V1Elts.push_back(Elt); 3495 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3496 ++V1InOrder; 3497 } else if (EltIdx == i+8) { 3498 V1Elts.push_back(Elt); 3499 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3500 ++V2InOrder; 3501 } else if (EltIdx < 8) { 3502 V1Elts.push_back(Elt); 3503 ++V1FromV1; 3504 } else { 3505 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3506 ++V2FromV2; 3507 } 3508 } 3509 3510 if (V2InOrder > V1InOrder) { 3511 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3512 std::swap(V1, V2); 3513 std::swap(V1Elts, V2Elts); 3514 std::swap(V1FromV1, V2FromV2); 3515 } 3516 3517 if ((V1FromV1 + V1InOrder) != 8) { 3518 // Some elements are from V2. 3519 if (V1FromV1) { 3520 // If there are elements that are from V1 but out of place, 3521 // then first sort them in place 3522 SmallVector<SDValue, 8> MaskVec; 3523 for (unsigned i = 0; i < 8; ++i) { 3524 SDValue Elt = V1Elts[i]; 3525 if (Elt.getOpcode() == ISD::UNDEF) { 3526 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3527 continue; 3528 } 3529 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3530 if (EltIdx >= 8) 3531 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3532 else 3533 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3534 } 3535 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3536 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3537 } 3538 3539 NewV = V1; 3540 for (unsigned i = 0; i < 8; ++i) { 3541 SDValue Elt = V1Elts[i]; 3542 if (Elt.getOpcode() == ISD::UNDEF) 3543 continue; 3544 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3545 if (EltIdx < 8) 3546 continue; 3547 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3548 DAG.getConstant(EltIdx - 8, PtrVT)); 3549 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3550 DAG.getConstant(i, PtrVT)); 3551 } 3552 return NewV; 3553 } else { 3554 // All elements are from V1. 3555 NewV = V1; 3556 for (unsigned i = 0; i < 8; ++i) { 3557 SDValue Elt = V1Elts[i]; 3558 if (Elt.getOpcode() == ISD::UNDEF) 3559 continue; 3560 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3561 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3562 DAG.getConstant(EltIdx, PtrVT)); 3563 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3564 DAG.getConstant(i, PtrVT)); 3565 } 3566 return NewV; 3567 } 3568} 3569 3570/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3571/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3572/// done when every pair / quad of shuffle mask elements point to elements in 3573/// the right sequence. e.g. 3574/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3575static 3576SDValue RewriteAsNarrowerShuffle(SDValue V1, SDValue V2, 3577 MVT VT, 3578 SDValue PermMask, SelectionDAG &DAG, 3579 TargetLowering &TLI) { 3580 unsigned NumElems = PermMask.getNumOperands(); 3581 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3582 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3583 MVT MaskEltVT = MaskVT.getVectorElementType(); 3584 MVT NewVT = MaskVT; 3585 switch (VT.getSimpleVT()) { 3586 default: assert(false && "Unexpected!"); 3587 case MVT::v4f32: NewVT = MVT::v2f64; break; 3588 case MVT::v4i32: NewVT = MVT::v2i64; break; 3589 case MVT::v8i16: NewVT = MVT::v4i32; break; 3590 case MVT::v16i8: NewVT = MVT::v4i32; break; 3591 } 3592 3593 if (NewWidth == 2) { 3594 if (VT.isInteger()) 3595 NewVT = MVT::v2i64; 3596 else 3597 NewVT = MVT::v2f64; 3598 } 3599 unsigned Scale = NumElems / NewWidth; 3600 SmallVector<SDValue, 8> MaskVec; 3601 for (unsigned i = 0; i < NumElems; i += Scale) { 3602 unsigned StartIdx = ~0U; 3603 for (unsigned j = 0; j < Scale; ++j) { 3604 SDValue Elt = PermMask.getOperand(i+j); 3605 if (Elt.getOpcode() == ISD::UNDEF) 3606 continue; 3607 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3608 if (StartIdx == ~0U) 3609 StartIdx = EltIdx - (EltIdx % Scale); 3610 if (EltIdx != StartIdx + j) 3611 return SDValue(); 3612 } 3613 if (StartIdx == ~0U) 3614 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEltVT)); 3615 else 3616 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MaskEltVT)); 3617 } 3618 3619 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3620 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3621 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3622 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3623 &MaskVec[0], MaskVec.size())); 3624} 3625 3626/// getVZextMovL - Return a zero-extending vector move low node. 3627/// 3628static SDValue getVZextMovL(MVT VT, MVT OpVT, 3629 SDValue SrcOp, SelectionDAG &DAG, 3630 const X86Subtarget *Subtarget) { 3631 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3632 LoadSDNode *LD = NULL; 3633 if (!isScalarLoadToVector(SrcOp.Val, &LD)) 3634 LD = dyn_cast<LoadSDNode>(SrcOp); 3635 if (!LD) { 3636 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3637 // instead. 3638 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3639 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3640 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3641 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3642 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3643 // PR2108 3644 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3645 return DAG.getNode(ISD::BIT_CONVERT, VT, 3646 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3647 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3648 SrcOp.getOperand(0).getOperand(0)))); 3649 } 3650 } 3651 } 3652 3653 return DAG.getNode(ISD::BIT_CONVERT, VT, 3654 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3655 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3656} 3657 3658/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of 3659/// shuffles. 3660static SDValue 3661LowerVECTOR_SHUFFLE_4wide(SDValue V1, SDValue V2, 3662 SDValue PermMask, MVT VT, SelectionDAG &DAG) { 3663 MVT MaskVT = PermMask.getValueType(); 3664 MVT MaskEVT = MaskVT.getVectorElementType(); 3665 SmallVector<std::pair<int, int>, 8> Locs; 3666 Locs.reserve(4); 3667 SmallVector<SDValue, 8> Mask1(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3668 unsigned NumHi = 0; 3669 unsigned NumLo = 0; 3670 for (unsigned i = 0; i != 4; ++i) { 3671 SDValue Elt = PermMask.getOperand(i); 3672 if (Elt.getOpcode() == ISD::UNDEF) { 3673 Locs[i] = std::make_pair(-1, -1); 3674 } else { 3675 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3676 assert(Val < 8 && "Invalid VECTOR_SHUFFLE index!"); 3677 if (Val < 4) { 3678 Locs[i] = std::make_pair(0, NumLo); 3679 Mask1[NumLo] = Elt; 3680 NumLo++; 3681 } else { 3682 Locs[i] = std::make_pair(1, NumHi); 3683 if (2+NumHi < 4) 3684 Mask1[2+NumHi] = Elt; 3685 NumHi++; 3686 } 3687 } 3688 } 3689 3690 if (NumLo <= 2 && NumHi <= 2) { 3691 // If no more than two elements come from either vector. This can be 3692 // implemented with two shuffles. First shuffle gather the elements. 3693 // The second shuffle, which takes the first shuffle as both of its 3694 // vector operands, put the elements into the right order. 3695 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3696 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3697 &Mask1[0], Mask1.size())); 3698 3699 SmallVector<SDValue, 8> Mask2(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3700 for (unsigned i = 0; i != 4; ++i) { 3701 if (Locs[i].first == -1) 3702 continue; 3703 else { 3704 unsigned Idx = (i < 2) ? 0 : 4; 3705 Idx += Locs[i].first * 2 + Locs[i].second; 3706 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3707 } 3708 } 3709 3710 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3711 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3712 &Mask2[0], Mask2.size())); 3713 } else if (NumLo == 3 || NumHi == 3) { 3714 // Otherwise, we must have three elements from one vector, call it X, and 3715 // one element from the other, call it Y. First, use a shufps to build an 3716 // intermediate vector with the one element from Y and the element from X 3717 // that will be in the same half in the final destination (the indexes don't 3718 // matter). Then, use a shufps to build the final vector, taking the half 3719 // containing the element from Y from the intermediate, and the other half 3720 // from X. 3721 if (NumHi == 3) { 3722 // Normalize it so the 3 elements come from V1. 3723 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3724 std::swap(V1, V2); 3725 } 3726 3727 // Find the element from V2. 3728 unsigned HiIndex; 3729 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 3730 SDValue Elt = PermMask.getOperand(HiIndex); 3731 if (Elt.getOpcode() == ISD::UNDEF) 3732 continue; 3733 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3734 if (Val >= 4) 3735 break; 3736 } 3737 3738 Mask1[0] = PermMask.getOperand(HiIndex); 3739 Mask1[1] = DAG.getNode(ISD::UNDEF, MaskEVT); 3740 Mask1[2] = PermMask.getOperand(HiIndex^1); 3741 Mask1[3] = DAG.getNode(ISD::UNDEF, MaskEVT); 3742 V2 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3743 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3744 3745 if (HiIndex >= 2) { 3746 Mask1[0] = PermMask.getOperand(0); 3747 Mask1[1] = PermMask.getOperand(1); 3748 Mask1[2] = DAG.getConstant(HiIndex & 1 ? 6 : 4, MaskEVT); 3749 Mask1[3] = DAG.getConstant(HiIndex & 1 ? 4 : 6, MaskEVT); 3750 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3751 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3752 } else { 3753 Mask1[0] = DAG.getConstant(HiIndex & 1 ? 2 : 0, MaskEVT); 3754 Mask1[1] = DAG.getConstant(HiIndex & 1 ? 0 : 2, MaskEVT); 3755 Mask1[2] = PermMask.getOperand(2); 3756 Mask1[3] = PermMask.getOperand(3); 3757 if (Mask1[2].getOpcode() != ISD::UNDEF) 3758 Mask1[2] = DAG.getConstant(cast<ConstantSDNode>(Mask1[2])->getValue()+4, 3759 MaskEVT); 3760 if (Mask1[3].getOpcode() != ISD::UNDEF) 3761 Mask1[3] = DAG.getConstant(cast<ConstantSDNode>(Mask1[3])->getValue()+4, 3762 MaskEVT); 3763 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, 3764 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3765 } 3766 } 3767 3768 // Break it into (shuffle shuffle_hi, shuffle_lo). 3769 Locs.clear(); 3770 SmallVector<SDValue,8> LoMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3771 SmallVector<SDValue,8> HiMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3772 SmallVector<SDValue,8> *MaskPtr = &LoMask; 3773 unsigned MaskIdx = 0; 3774 unsigned LoIdx = 0; 3775 unsigned HiIdx = 2; 3776 for (unsigned i = 0; i != 4; ++i) { 3777 if (i == 2) { 3778 MaskPtr = &HiMask; 3779 MaskIdx = 1; 3780 LoIdx = 0; 3781 HiIdx = 2; 3782 } 3783 SDValue Elt = PermMask.getOperand(i); 3784 if (Elt.getOpcode() == ISD::UNDEF) { 3785 Locs[i] = std::make_pair(-1, -1); 3786 } else if (cast<ConstantSDNode>(Elt)->getValue() < 4) { 3787 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3788 (*MaskPtr)[LoIdx] = Elt; 3789 LoIdx++; 3790 } else { 3791 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3792 (*MaskPtr)[HiIdx] = Elt; 3793 HiIdx++; 3794 } 3795 } 3796 3797 SDValue LoShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3798 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3799 &LoMask[0], LoMask.size())); 3800 SDValue HiShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3801 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3802 &HiMask[0], HiMask.size())); 3803 SmallVector<SDValue, 8> MaskOps; 3804 for (unsigned i = 0; i != 4; ++i) { 3805 if (Locs[i].first == -1) { 3806 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3807 } else { 3808 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 3809 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3810 } 3811 } 3812 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3813 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3814 &MaskOps[0], MaskOps.size())); 3815} 3816 3817SDValue 3818X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 3819 SDValue V1 = Op.getOperand(0); 3820 SDValue V2 = Op.getOperand(1); 3821 SDValue PermMask = Op.getOperand(2); 3822 MVT VT = Op.getValueType(); 3823 unsigned NumElems = PermMask.getNumOperands(); 3824 bool isMMX = VT.getSizeInBits() == 64; 3825 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3826 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3827 bool V1IsSplat = false; 3828 bool V2IsSplat = false; 3829 3830 if (isUndefShuffle(Op.Val)) 3831 return DAG.getNode(ISD::UNDEF, VT); 3832 3833 if (isZeroShuffle(Op.Val)) 3834 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3835 3836 if (isIdentityMask(PermMask.Val)) 3837 return V1; 3838 else if (isIdentityMask(PermMask.Val, true)) 3839 return V2; 3840 3841 if (isSplatMask(PermMask.Val)) { 3842 if (isMMX || NumElems < 4) return Op; 3843 // Promote it to a v4{if}32 splat. 3844 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3845 } 3846 3847 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3848 // do it! 3849 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3850 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3851 if (NewOp.Val) 3852 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3853 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3854 // FIXME: Figure out a cleaner way to do this. 3855 // Try to make use of movq to zero out the top part. 3856 if (ISD::isBuildVectorAllZeros(V2.Val)) { 3857 SDValue NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3858 DAG, *this); 3859 if (NewOp.Val) { 3860 SDValue NewV1 = NewOp.getOperand(0); 3861 SDValue NewV2 = NewOp.getOperand(1); 3862 SDValue NewMask = NewOp.getOperand(2); 3863 if (isCommutedMOVL(NewMask.Val, true, false)) { 3864 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3865 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3866 } 3867 } 3868 } else if (ISD::isBuildVectorAllZeros(V1.Val)) { 3869 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3870 DAG, *this); 3871 if (NewOp.Val && X86::isMOVLMask(NewOp.getOperand(2).Val)) 3872 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3873 DAG, Subtarget); 3874 } 3875 } 3876 3877 // Check if this can be converted into a logical shift. 3878 bool isLeft = false; 3879 unsigned ShAmt = 0; 3880 SDValue ShVal; 3881 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3882 if (isShift && ShVal.hasOneUse()) { 3883 // If the shifted value has multiple uses, it may be cheaper to use 3884 // v_set0 + movlhps or movhlps, etc. 3885 MVT EVT = VT.getVectorElementType(); 3886 ShAmt *= EVT.getSizeInBits(); 3887 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3888 } 3889 3890 if (X86::isMOVLMask(PermMask.Val)) { 3891 if (V1IsUndef) 3892 return V2; 3893 if (ISD::isBuildVectorAllZeros(V1.Val)) 3894 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3895 if (!isMMX) 3896 return Op; 3897 } 3898 3899 if (!isMMX && (X86::isMOVSHDUPMask(PermMask.Val) || 3900 X86::isMOVSLDUPMask(PermMask.Val) || 3901 X86::isMOVHLPSMask(PermMask.Val) || 3902 X86::isMOVHPMask(PermMask.Val) || 3903 X86::isMOVLPMask(PermMask.Val))) 3904 return Op; 3905 3906 if (ShouldXformToMOVHLPS(PermMask.Val) || 3907 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val)) 3908 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3909 3910 if (isShift) { 3911 // No better options. Use a vshl / vsrl. 3912 MVT EVT = VT.getVectorElementType(); 3913 ShAmt *= EVT.getSizeInBits(); 3914 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3915 } 3916 3917 bool Commuted = false; 3918 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3919 // 1,1,1,1 -> v8i16 though. 3920 V1IsSplat = isSplatVector(V1.Val); 3921 V2IsSplat = isSplatVector(V2.Val); 3922 3923 // Canonicalize the splat or undef, if present, to be on the RHS. 3924 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3925 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3926 std::swap(V1IsSplat, V2IsSplat); 3927 std::swap(V1IsUndef, V2IsUndef); 3928 Commuted = true; 3929 } 3930 3931 // FIXME: Figure out a cleaner way to do this. 3932 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) { 3933 if (V2IsUndef) return V1; 3934 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3935 if (V2IsSplat) { 3936 // V2 is a splat, so the mask may be malformed. That is, it may point 3937 // to any V2 element. The instruction selectior won't like this. Get 3938 // a corrected mask and commute to form a proper MOVS{S|D}. 3939 SDValue NewMask = getMOVLMask(NumElems, DAG); 3940 if (NewMask.Val != PermMask.Val) 3941 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3942 } 3943 return Op; 3944 } 3945 3946 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3947 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3948 X86::isUNPCKLMask(PermMask.Val) || 3949 X86::isUNPCKHMask(PermMask.Val)) 3950 return Op; 3951 3952 if (V2IsSplat) { 3953 // Normalize mask so all entries that point to V2 points to its first 3954 // element then try to match unpck{h|l} again. If match, return a 3955 // new vector_shuffle with the corrected mask. 3956 SDValue NewMask = NormalizeMask(PermMask, DAG); 3957 if (NewMask.Val != PermMask.Val) { 3958 if (X86::isUNPCKLMask(PermMask.Val, true)) { 3959 SDValue NewMask = getUnpacklMask(NumElems, DAG); 3960 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3961 } else if (X86::isUNPCKHMask(PermMask.Val, true)) { 3962 SDValue NewMask = getUnpackhMask(NumElems, DAG); 3963 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 3964 } 3965 } 3966 } 3967 3968 // Normalize the node to match x86 shuffle ops if needed 3969 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val)) 3970 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3971 3972 if (Commuted) { 3973 // Commute is back and try unpck* again. 3974 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3975 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || 3976 X86::isUNPCKH_v_undef_Mask(PermMask.Val) || 3977 X86::isUNPCKLMask(PermMask.Val) || 3978 X86::isUNPCKHMask(PermMask.Val)) 3979 return Op; 3980 } 3981 3982 // Try PSHUF* first, then SHUFP*. 3983 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 3984 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 3985 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.Val)) { 3986 if (V2.getOpcode() != ISD::UNDEF) 3987 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 3988 DAG.getNode(ISD::UNDEF, VT), PermMask); 3989 return Op; 3990 } 3991 3992 if (!isMMX) { 3993 if (Subtarget->hasSSE2() && 3994 (X86::isPSHUFDMask(PermMask.Val) || 3995 X86::isPSHUFHWMask(PermMask.Val) || 3996 X86::isPSHUFLWMask(PermMask.Val))) { 3997 MVT RVT = VT; 3998 if (VT == MVT::v4f32) { 3999 RVT = MVT::v4i32; 4000 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 4001 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 4002 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4003 } else if (V2.getOpcode() != ISD::UNDEF) 4004 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 4005 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4006 if (RVT != VT) 4007 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 4008 return Op; 4009 } 4010 4011 // Binary or unary shufps. 4012 if (X86::isSHUFPMask(PermMask.Val) || 4013 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.Val))) 4014 return Op; 4015 } 4016 4017 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 4018 if (VT == MVT::v8i16) { 4019 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 4020 if (NewOp.Val) 4021 return NewOp; 4022 } 4023 4024 // Handle all 4 wide cases with a number of shuffles except for MMX. 4025 if (NumElems == 4 && !isMMX) 4026 return LowerVECTOR_SHUFFLE_4wide(V1, V2, PermMask, VT, DAG); 4027 4028 return SDValue(); 4029} 4030 4031SDValue 4032X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 4033 SelectionDAG &DAG) { 4034 MVT VT = Op.getValueType(); 4035 if (VT.getSizeInBits() == 8) { 4036 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 4037 Op.getOperand(0), Op.getOperand(1)); 4038 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4039 DAG.getValueType(VT)); 4040 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4041 } else if (VT.getSizeInBits() == 16) { 4042 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 4043 Op.getOperand(0), Op.getOperand(1)); 4044 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4045 DAG.getValueType(VT)); 4046 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4047 } else if (VT == MVT::f32) { 4048 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 4049 // the result back to FR32 register. It's only worth matching if the 4050 // result has a single use which is a store or a bitcast to i32. 4051 if (!Op.hasOneUse()) 4052 return SDValue(); 4053 SDNode *User = *Op.Val->use_begin(); 4054 if (User->getOpcode() != ISD::STORE && 4055 (User->getOpcode() != ISD::BIT_CONVERT || 4056 User->getValueType(0) != MVT::i32)) 4057 return SDValue(); 4058 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4059 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 4060 Op.getOperand(1)); 4061 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 4062 } 4063 return SDValue(); 4064} 4065 4066 4067SDValue 4068X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4069 if (!isa<ConstantSDNode>(Op.getOperand(1))) 4070 return SDValue(); 4071 4072 if (Subtarget->hasSSE41()) { 4073 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 4074 if (Res.Val) 4075 return Res; 4076 } 4077 4078 MVT VT = Op.getValueType(); 4079 // TODO: handle v16i8. 4080 if (VT.getSizeInBits() == 16) { 4081 SDValue Vec = Op.getOperand(0); 4082 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4083 if (Idx == 0) 4084 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4085 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4086 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4087 Op.getOperand(1))); 4088 // Transform it so it match pextrw which produces a 32-bit result. 4089 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4090 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4091 Op.getOperand(0), Op.getOperand(1)); 4092 SDValue Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4093 DAG.getValueType(VT)); 4094 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4095 } else if (VT.getSizeInBits() == 32) { 4096 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4097 if (Idx == 0) 4098 return Op; 4099 // SHUFPS the element to the lowest double word, then movss. 4100 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4101 SmallVector<SDValue, 8> IdxVec; 4102 IdxVec. 4103 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4104 IdxVec. 4105 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4106 IdxVec. 4107 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4108 IdxVec. 4109 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4110 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4111 &IdxVec[0], IdxVec.size()); 4112 SDValue Vec = Op.getOperand(0); 4113 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4114 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4115 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4116 DAG.getIntPtrConstant(0)); 4117 } else if (VT.getSizeInBits() == 64) { 4118 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4119 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4120 // to match extract_elt for f64. 4121 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4122 if (Idx == 0) 4123 return Op; 4124 4125 // UNPCKHPD the element to the lowest double word, then movsd. 4126 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4127 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4128 MVT MaskVT = MVT::getIntVectorWithNumElements(2); 4129 SmallVector<SDValue, 8> IdxVec; 4130 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4131 IdxVec. 4132 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4133 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4134 &IdxVec[0], IdxVec.size()); 4135 SDValue Vec = Op.getOperand(0); 4136 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4137 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4139 DAG.getIntPtrConstant(0)); 4140 } 4141 4142 return SDValue(); 4143} 4144 4145SDValue 4146X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){ 4147 MVT VT = Op.getValueType(); 4148 MVT EVT = VT.getVectorElementType(); 4149 4150 SDValue N0 = Op.getOperand(0); 4151 SDValue N1 = Op.getOperand(1); 4152 SDValue N2 = Op.getOperand(2); 4153 4154 if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) && 4155 isa<ConstantSDNode>(N2)) { 4156 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4157 : X86ISD::PINSRW; 4158 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4159 // argument. 4160 if (N1.getValueType() != MVT::i32) 4161 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4162 if (N2.getValueType() != MVT::i32) 4163 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4164 return DAG.getNode(Opc, VT, N0, N1, N2); 4165 } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 4166 // Bits [7:6] of the constant are the source select. This will always be 4167 // zero here. The DAG Combiner may combine an extract_elt index into these 4168 // bits. For example (insert (extract, 3), 2) could be matched by putting 4169 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4170 // Bits [5:4] of the constant are the destination select. This is the 4171 // value of the incoming immediate. 4172 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4173 // combine either bitwise AND or insert of float 0.0 to set these bits. 4174 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4175 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4176 } 4177 return SDValue(); 4178} 4179 4180SDValue 4181X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4182 MVT VT = Op.getValueType(); 4183 MVT EVT = VT.getVectorElementType(); 4184 4185 if (Subtarget->hasSSE41()) 4186 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4187 4188 if (EVT == MVT::i8) 4189 return SDValue(); 4190 4191 SDValue N0 = Op.getOperand(0); 4192 SDValue N1 = Op.getOperand(1); 4193 SDValue N2 = Op.getOperand(2); 4194 4195 if (EVT.getSizeInBits() == 16) { 4196 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4197 // as its second argument. 4198 if (N1.getValueType() != MVT::i32) 4199 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4200 if (N2.getValueType() != MVT::i32) 4201 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4202 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4203 } 4204 return SDValue(); 4205} 4206 4207SDValue 4208X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) { 4209 if (Op.getValueType() == MVT::v2f32) 4210 return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f32, 4211 DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i32, 4212 DAG.getNode(ISD::BIT_CONVERT, MVT::i32, 4213 Op.getOperand(0)))); 4214 4215 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4216 MVT VT = MVT::v2i32; 4217 switch (Op.getValueType().getSimpleVT()) { 4218 default: break; 4219 case MVT::v16i8: 4220 case MVT::v8i16: 4221 VT = MVT::v4i32; 4222 break; 4223 } 4224 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4225 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4226} 4227 4228// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4229// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4230// one of the above mentioned nodes. It has to be wrapped because otherwise 4231// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4232// be used to form addressing mode. These wrapped nodes will be selected 4233// into MOV32ri. 4234SDValue 4235X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) { 4236 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4237 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), 4238 getPointerTy(), 4239 CP->getAlignment()); 4240 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4241 // With PIC, the address is actually $g + Offset. 4242 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4243 !Subtarget->isPICStyleRIPRel()) { 4244 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4245 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4246 Result); 4247 } 4248 4249 return Result; 4250} 4251 4252SDValue 4253X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) { 4254 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4255 SDValue Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4256 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4257 // With PIC, the address is actually $g + Offset. 4258 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4259 !Subtarget->isPICStyleRIPRel()) { 4260 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4261 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4262 Result); 4263 } 4264 4265 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4266 // load the value at address GV, not the value of GV itself. This means that 4267 // the GlobalAddress must be in the base or index register of the address, not 4268 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4269 // The same applies for external symbols during PIC codegen 4270 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4271 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4272 PseudoSourceValue::getGOT(), 0); 4273 4274 return Result; 4275} 4276 4277// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4278static SDValue 4279LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4280 const MVT PtrVT) { 4281 SDValue InFlag; 4282 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4283 DAG.getNode(X86ISD::GlobalBaseReg, 4284 PtrVT), InFlag); 4285 InFlag = Chain.getValue(1); 4286 4287 // emit leal symbol@TLSGD(,%ebx,1), %eax 4288 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4289 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4290 GA->getValueType(0), 4291 GA->getOffset()); 4292 SDValue Ops[] = { Chain, TGA, InFlag }; 4293 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4294 InFlag = Result.getValue(2); 4295 Chain = Result.getValue(1); 4296 4297 // call ___tls_get_addr. This function receives its argument in 4298 // the register EAX. 4299 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4300 InFlag = Chain.getValue(1); 4301 4302 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4303 SDValue Ops1[] = { Chain, 4304 DAG.getTargetExternalSymbol("___tls_get_addr", 4305 PtrVT), 4306 DAG.getRegister(X86::EAX, PtrVT), 4307 DAG.getRegister(X86::EBX, PtrVT), 4308 InFlag }; 4309 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4310 InFlag = Chain.getValue(1); 4311 4312 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4313} 4314 4315// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4316static SDValue 4317LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4318 const MVT PtrVT) { 4319 SDValue InFlag, Chain; 4320 4321 // emit leaq symbol@TLSGD(%rip), %rdi 4322 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4323 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4324 GA->getValueType(0), 4325 GA->getOffset()); 4326 SDValue Ops[] = { DAG.getEntryNode(), TGA}; 4327 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4328 Chain = Result.getValue(1); 4329 InFlag = Result.getValue(2); 4330 4331 // call __tls_get_addr. This function receives its argument in 4332 // the register RDI. 4333 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4334 InFlag = Chain.getValue(1); 4335 4336 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4337 SDValue Ops1[] = { Chain, 4338 DAG.getTargetExternalSymbol("__tls_get_addr", 4339 PtrVT), 4340 DAG.getRegister(X86::RDI, PtrVT), 4341 InFlag }; 4342 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4343 InFlag = Chain.getValue(1); 4344 4345 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4346} 4347 4348// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4349// "local exec" model. 4350static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4351 const MVT PtrVT) { 4352 // Get the Thread Pointer 4353 SDValue ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4354 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4355 // exec) 4356 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4357 GA->getValueType(0), 4358 GA->getOffset()); 4359 SDValue Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4360 4361 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4362 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4363 PseudoSourceValue::getGOT(), 0); 4364 4365 // The address of the thread local variable is the add of the thread 4366 // pointer with the offset of the variable. 4367 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4368} 4369 4370SDValue 4371X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) { 4372 // TODO: implement the "local dynamic" model 4373 // TODO: implement the "initial exec"model for pic executables 4374 assert(Subtarget->isTargetELF() && 4375 "TLS not implemented for non-ELF targets"); 4376 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4377 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4378 // otherwise use the "Local Exec"TLS Model 4379 if (Subtarget->is64Bit()) { 4380 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4381 } else { 4382 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4383 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4384 else 4385 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4386 } 4387} 4388 4389SDValue 4390X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) { 4391 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4392 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4393 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4394 // With PIC, the address is actually $g + Offset. 4395 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4396 !Subtarget->isPICStyleRIPRel()) { 4397 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4398 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4399 Result); 4400 } 4401 4402 return Result; 4403} 4404 4405SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) { 4406 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4407 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4408 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4409 // With PIC, the address is actually $g + Offset. 4410 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4411 !Subtarget->isPICStyleRIPRel()) { 4412 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4413 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4414 Result); 4415 } 4416 4417 return Result; 4418} 4419 4420/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4421/// take a 2 x i32 value to shift plus a shift amount. 4422SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) { 4423 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4424 MVT VT = Op.getValueType(); 4425 unsigned VTBits = VT.getSizeInBits(); 4426 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4427 SDValue ShOpLo = Op.getOperand(0); 4428 SDValue ShOpHi = Op.getOperand(1); 4429 SDValue ShAmt = Op.getOperand(2); 4430 SDValue Tmp1 = isSRA ? 4431 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4432 DAG.getConstant(0, VT); 4433 4434 SDValue Tmp2, Tmp3; 4435 if (Op.getOpcode() == ISD::SHL_PARTS) { 4436 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4437 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4438 } else { 4439 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4440 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4441 } 4442 4443 SDValue AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4444 DAG.getConstant(VTBits, MVT::i8)); 4445 SDValue Cond = DAG.getNode(X86ISD::CMP, VT, 4446 AndNode, DAG.getConstant(0, MVT::i8)); 4447 4448 SDValue Hi, Lo; 4449 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4450 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 4451 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 4452 4453 if (Op.getOpcode() == ISD::SHL_PARTS) { 4454 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4455 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4456 } else { 4457 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4458 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4459 } 4460 4461 SDValue Ops[2] = { Lo, Hi }; 4462 return DAG.getMergeValues(Ops, 2); 4463} 4464 4465SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 4466 MVT SrcVT = Op.getOperand(0).getValueType(); 4467 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4468 "Unknown SINT_TO_FP to lower!"); 4469 4470 // These are really Legal; caller falls through into that case. 4471 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4472 return SDValue(); 4473 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4474 Subtarget->is64Bit()) 4475 return SDValue(); 4476 4477 unsigned Size = SrcVT.getSizeInBits()/8; 4478 MachineFunction &MF = DAG.getMachineFunction(); 4479 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4480 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4481 SDValue Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4482 StackSlot, 4483 PseudoSourceValue::getFixedStack(SSFI), 0); 4484 4485 // Build the FILD 4486 SDVTList Tys; 4487 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4488 if (useSSE) 4489 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4490 else 4491 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4492 SmallVector<SDValue, 8> Ops; 4493 Ops.push_back(Chain); 4494 Ops.push_back(StackSlot); 4495 Ops.push_back(DAG.getValueType(SrcVT)); 4496 SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4497 Tys, &Ops[0], Ops.size()); 4498 4499 if (useSSE) { 4500 Chain = Result.getValue(1); 4501 SDValue InFlag = Result.getValue(2); 4502 4503 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4504 // shouldn't be necessary except that RFP cannot be live across 4505 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4506 MachineFunction &MF = DAG.getMachineFunction(); 4507 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4508 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4509 Tys = DAG.getVTList(MVT::Other); 4510 SmallVector<SDValue, 8> Ops; 4511 Ops.push_back(Chain); 4512 Ops.push_back(Result); 4513 Ops.push_back(StackSlot); 4514 Ops.push_back(DAG.getValueType(Op.getValueType())); 4515 Ops.push_back(InFlag); 4516 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4517 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4518 PseudoSourceValue::getFixedStack(SSFI), 0); 4519 } 4520 4521 return Result; 4522} 4523 4524std::pair<SDValue,SDValue> X86TargetLowering:: 4525FP_TO_SINTHelper(SDValue Op, SelectionDAG &DAG) { 4526 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4527 Op.getValueType().getSimpleVT() >= MVT::i16 && 4528 "Unknown FP_TO_SINT to lower!"); 4529 4530 // These are really Legal. 4531 if (Op.getValueType() == MVT::i32 && 4532 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4533 return std::make_pair(SDValue(), SDValue()); 4534 if (Subtarget->is64Bit() && 4535 Op.getValueType() == MVT::i64 && 4536 Op.getOperand(0).getValueType() != MVT::f80) 4537 return std::make_pair(SDValue(), SDValue()); 4538 4539 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4540 // stack slot. 4541 MachineFunction &MF = DAG.getMachineFunction(); 4542 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4543 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4544 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4545 unsigned Opc; 4546 switch (Op.getValueType().getSimpleVT()) { 4547 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4548 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4549 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4550 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4551 } 4552 4553 SDValue Chain = DAG.getEntryNode(); 4554 SDValue Value = Op.getOperand(0); 4555 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4556 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4557 Chain = DAG.getStore(Chain, Value, StackSlot, 4558 PseudoSourceValue::getFixedStack(SSFI), 0); 4559 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4560 SDValue Ops[] = { 4561 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4562 }; 4563 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4564 Chain = Value.getValue(1); 4565 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4566 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4567 } 4568 4569 // Build the FP_TO_INT*_IN_MEM 4570 SDValue Ops[] = { Chain, Value, StackSlot }; 4571 SDValue FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4572 4573 return std::make_pair(FIST, StackSlot); 4574} 4575 4576SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) { 4577 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(Op, DAG); 4578 SDValue FIST = Vals.first, StackSlot = Vals.second; 4579 if (FIST.Val == 0) return SDValue(); 4580 4581 // Load the result. 4582 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4583} 4584 4585SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4586 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG); 4587 SDValue FIST = Vals.first, StackSlot = Vals.second; 4588 if (FIST.Val == 0) return 0; 4589 4590 MVT VT = N->getValueType(0); 4591 4592 // Return a load from the stack slot. 4593 SDValue Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0); 4594 4595 // Use MERGE_VALUES to drop the chain result value and get a node with one 4596 // result. This requires turning off getMergeValues simplification, since 4597 // otherwise it will give us Res back. 4598 return DAG.getMergeValues(&Res, 1, false).Val; 4599} 4600 4601SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) { 4602 MVT VT = Op.getValueType(); 4603 MVT EltVT = VT; 4604 if (VT.isVector()) 4605 EltVT = VT.getVectorElementType(); 4606 std::vector<Constant*> CV; 4607 if (EltVT == MVT::f64) { 4608 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4609 CV.push_back(C); 4610 CV.push_back(C); 4611 } else { 4612 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4613 CV.push_back(C); 4614 CV.push_back(C); 4615 CV.push_back(C); 4616 CV.push_back(C); 4617 } 4618 Constant *C = ConstantVector::get(CV); 4619 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4620 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4621 PseudoSourceValue::getConstantPool(), 0, 4622 false, 16); 4623 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4624} 4625 4626SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) { 4627 MVT VT = Op.getValueType(); 4628 MVT EltVT = VT; 4629 unsigned EltNum = 1; 4630 if (VT.isVector()) { 4631 EltVT = VT.getVectorElementType(); 4632 EltNum = VT.getVectorNumElements(); 4633 } 4634 std::vector<Constant*> CV; 4635 if (EltVT == MVT::f64) { 4636 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4637 CV.push_back(C); 4638 CV.push_back(C); 4639 } else { 4640 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4641 CV.push_back(C); 4642 CV.push_back(C); 4643 CV.push_back(C); 4644 CV.push_back(C); 4645 } 4646 Constant *C = ConstantVector::get(CV); 4647 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4648 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4649 PseudoSourceValue::getConstantPool(), 0, 4650 false, 16); 4651 if (VT.isVector()) { 4652 return DAG.getNode(ISD::BIT_CONVERT, VT, 4653 DAG.getNode(ISD::XOR, MVT::v2i64, 4654 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4655 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4656 } else { 4657 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4658 } 4659} 4660 4661SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { 4662 SDValue Op0 = Op.getOperand(0); 4663 SDValue Op1 = Op.getOperand(1); 4664 MVT VT = Op.getValueType(); 4665 MVT SrcVT = Op1.getValueType(); 4666 4667 // If second operand is smaller, extend it first. 4668 if (SrcVT.bitsLT(VT)) { 4669 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4670 SrcVT = VT; 4671 } 4672 // And if it is bigger, shrink it first. 4673 if (SrcVT.bitsGT(VT)) { 4674 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4675 SrcVT = VT; 4676 } 4677 4678 // At this point the operands and the result should have the same 4679 // type, and that won't be f80 since that is not custom lowered. 4680 4681 // First get the sign bit of second operand. 4682 std::vector<Constant*> CV; 4683 if (SrcVT == MVT::f64) { 4684 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4685 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4686 } else { 4687 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4688 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4689 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4690 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4691 } 4692 Constant *C = ConstantVector::get(CV); 4693 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4694 SDValue Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4695 PseudoSourceValue::getConstantPool(), 0, 4696 false, 16); 4697 SDValue SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4698 4699 // Shift sign bit right or left if the two operands have different types. 4700 if (SrcVT.bitsGT(VT)) { 4701 // Op0 is MVT::f32, Op1 is MVT::f64. 4702 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4703 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4704 DAG.getConstant(32, MVT::i32)); 4705 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4706 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4707 DAG.getIntPtrConstant(0)); 4708 } 4709 4710 // Clear first operand sign bit. 4711 CV.clear(); 4712 if (VT == MVT::f64) { 4713 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4714 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4715 } else { 4716 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4717 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4718 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4719 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4720 } 4721 C = ConstantVector::get(CV); 4722 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4723 SDValue Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4724 PseudoSourceValue::getConstantPool(), 0, 4725 false, 16); 4726 SDValue Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4727 4728 // Or the value with the sign bit. 4729 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4730} 4731 4732SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) { 4733 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4734 SDValue Cond; 4735 SDValue Op0 = Op.getOperand(0); 4736 SDValue Op1 = Op.getOperand(1); 4737 SDValue CC = Op.getOperand(2); 4738 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4739 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4740 unsigned X86CC; 4741 4742 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4743 Op0, Op1, DAG)) { 4744 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4745 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4746 DAG.getConstant(X86CC, MVT::i8), Cond); 4747 } 4748 4749 assert(isFP && "Illegal integer SetCC!"); 4750 4751 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4752 switch (SetCCOpcode) { 4753 default: assert(false && "Illegal floating point SetCC!"); 4754 case ISD::SETOEQ: { // !PF & ZF 4755 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4756 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4757 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4758 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4759 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4760 } 4761 case ISD::SETUNE: { // PF | !ZF 4762 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4763 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4764 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4765 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4766 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4767 } 4768 } 4769} 4770 4771SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 4772 SDValue Cond; 4773 SDValue Op0 = Op.getOperand(0); 4774 SDValue Op1 = Op.getOperand(1); 4775 SDValue CC = Op.getOperand(2); 4776 MVT VT = Op.getValueType(); 4777 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4778 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4779 4780 if (isFP) { 4781 unsigned SSECC = 8; 4782 MVT VT0 = Op0.getValueType(); 4783 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); 4784 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; 4785 bool Swap = false; 4786 4787 switch (SetCCOpcode) { 4788 default: break; 4789 case ISD::SETOEQ: 4790 case ISD::SETEQ: SSECC = 0; break; 4791 case ISD::SETOGT: 4792 case ISD::SETGT: Swap = true; // Fallthrough 4793 case ISD::SETLT: 4794 case ISD::SETOLT: SSECC = 1; break; 4795 case ISD::SETOGE: 4796 case ISD::SETGE: Swap = true; // Fallthrough 4797 case ISD::SETLE: 4798 case ISD::SETOLE: SSECC = 2; break; 4799 case ISD::SETUO: SSECC = 3; break; 4800 case ISD::SETUNE: 4801 case ISD::SETNE: SSECC = 4; break; 4802 case ISD::SETULE: Swap = true; 4803 case ISD::SETUGE: SSECC = 5; break; 4804 case ISD::SETULT: Swap = true; 4805 case ISD::SETUGT: SSECC = 6; break; 4806 case ISD::SETO: SSECC = 7; break; 4807 } 4808 if (Swap) 4809 std::swap(Op0, Op1); 4810 4811 // In the two special cases we can't handle, emit two comparisons. 4812 if (SSECC == 8) { 4813 if (SetCCOpcode == ISD::SETUEQ) { 4814 SDValue UNORD, EQ; 4815 UNORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 4816 EQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 4817 return DAG.getNode(ISD::OR, VT, UNORD, EQ); 4818 } 4819 else if (SetCCOpcode == ISD::SETONE) { 4820 SDValue ORD, NEQ; 4821 ORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 4822 NEQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 4823 return DAG.getNode(ISD::AND, VT, ORD, NEQ); 4824 } 4825 assert(0 && "Illegal FP comparison"); 4826 } 4827 // Handle all other FP comparisons here. 4828 return DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 4829 } 4830 4831 // We are handling one of the integer comparisons here. Since SSE only has 4832 // GT and EQ comparisons for integer, swapping operands and multiple 4833 // operations may be required for some comparisons. 4834 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 4835 bool Swap = false, Invert = false, FlipSigns = false; 4836 4837 switch (VT.getSimpleVT()) { 4838 default: break; 4839 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 4840 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 4841 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 4842 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 4843 } 4844 4845 switch (SetCCOpcode) { 4846 default: break; 4847 case ISD::SETNE: Invert = true; 4848 case ISD::SETEQ: Opc = EQOpc; break; 4849 case ISD::SETLT: Swap = true; 4850 case ISD::SETGT: Opc = GTOpc; break; 4851 case ISD::SETGE: Swap = true; 4852 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 4853 case ISD::SETULT: Swap = true; 4854 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 4855 case ISD::SETUGE: Swap = true; 4856 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 4857 } 4858 if (Swap) 4859 std::swap(Op0, Op1); 4860 4861 // Since SSE has no unsigned integer comparisons, we need to flip the sign 4862 // bits of the inputs before performing those operations. 4863 if (FlipSigns) { 4864 MVT EltVT = VT.getVectorElementType(); 4865 SDValue SignBit = DAG.getConstant(EltVT.getIntegerVTSignBit(), EltVT); 4866 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 4867 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, VT, &SignBits[0], 4868 SignBits.size()); 4869 Op0 = DAG.getNode(ISD::XOR, VT, Op0, SignVec); 4870 Op1 = DAG.getNode(ISD::XOR, VT, Op1, SignVec); 4871 } 4872 4873 SDValue Result = DAG.getNode(Opc, VT, Op0, Op1); 4874 4875 // If the logical-not of the result is required, perform that now. 4876 if (Invert) { 4877 MVT EltVT = VT.getVectorElementType(); 4878 SDValue NegOne = DAG.getConstant(EltVT.getIntegerVTBitMask(), EltVT); 4879 std::vector<SDValue> NegOnes(VT.getVectorNumElements(), NegOne); 4880 SDValue NegOneV = DAG.getNode(ISD::BUILD_VECTOR, VT, &NegOnes[0], 4881 NegOnes.size()); 4882 Result = DAG.getNode(ISD::XOR, VT, Result, NegOneV); 4883 } 4884 return Result; 4885} 4886 4887SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) { 4888 bool addTest = true; 4889 SDValue Cond = Op.getOperand(0); 4890 SDValue CC; 4891 4892 if (Cond.getOpcode() == ISD::SETCC) 4893 Cond = LowerSETCC(Cond, DAG); 4894 4895 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4896 // setting operand in place of the X86ISD::SETCC. 4897 if (Cond.getOpcode() == X86ISD::SETCC) { 4898 CC = Cond.getOperand(0); 4899 4900 SDValue Cmp = Cond.getOperand(1); 4901 unsigned Opc = Cmp.getOpcode(); 4902 MVT VT = Op.getValueType(); 4903 4904 bool IllegalFPCMov = false; 4905 if (VT.isFloatingPoint() && !VT.isVector() && 4906 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4907 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4908 4909 if ((Opc == X86ISD::CMP || 4910 Opc == X86ISD::COMI || 4911 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4912 Cond = Cmp; 4913 addTest = false; 4914 } 4915 } 4916 4917 if (addTest) { 4918 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4919 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4920 } 4921 4922 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4923 MVT::Flag); 4924 SmallVector<SDValue, 4> Ops; 4925 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4926 // condition is true. 4927 Ops.push_back(Op.getOperand(2)); 4928 Ops.push_back(Op.getOperand(1)); 4929 Ops.push_back(CC); 4930 Ops.push_back(Cond); 4931 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4932} 4933 4934SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) { 4935 bool addTest = true; 4936 SDValue Chain = Op.getOperand(0); 4937 SDValue Cond = Op.getOperand(1); 4938 SDValue Dest = Op.getOperand(2); 4939 SDValue CC; 4940 4941 if (Cond.getOpcode() == ISD::SETCC) 4942 Cond = LowerSETCC(Cond, DAG); 4943 4944 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4945 // setting operand in place of the X86ISD::SETCC. 4946 if (Cond.getOpcode() == X86ISD::SETCC) { 4947 CC = Cond.getOperand(0); 4948 4949 SDValue Cmp = Cond.getOperand(1); 4950 unsigned Opc = Cmp.getOpcode(); 4951 if (Opc == X86ISD::CMP || 4952 Opc == X86ISD::COMI || 4953 Opc == X86ISD::UCOMI) { 4954 Cond = Cmp; 4955 addTest = false; 4956 } 4957 } 4958 4959 if (addTest) { 4960 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4961 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4962 } 4963 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 4964 Chain, Op.getOperand(2), CC, Cond); 4965} 4966 4967 4968// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 4969// Calls to _alloca is needed to probe the stack when allocating more than 4k 4970// bytes in one go. Touching the stack at 4K increments is necessary to ensure 4971// that the guard pages used by the OS virtual memory manager are allocated in 4972// correct sequence. 4973SDValue 4974X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 4975 SelectionDAG &DAG) { 4976 assert(Subtarget->isTargetCygMing() && 4977 "This should be used only on Cygwin/Mingw targets"); 4978 4979 // Get the inputs. 4980 SDValue Chain = Op.getOperand(0); 4981 SDValue Size = Op.getOperand(1); 4982 // FIXME: Ensure alignment here 4983 4984 SDValue Flag; 4985 4986 MVT IntPtr = getPointerTy(); 4987 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 4988 4989 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 4990 4991 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 4992 Flag = Chain.getValue(1); 4993 4994 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4995 SDValue Ops[] = { Chain, 4996 DAG.getTargetExternalSymbol("_alloca", IntPtr), 4997 DAG.getRegister(X86::EAX, IntPtr), 4998 DAG.getRegister(X86StackPtr, SPTy), 4999 Flag }; 5000 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 5001 Flag = Chain.getValue(1); 5002 5003 Chain = DAG.getCALLSEQ_END(Chain, 5004 DAG.getIntPtrConstant(0), 5005 DAG.getIntPtrConstant(0), 5006 Flag); 5007 5008 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 5009 5010 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 5011 return DAG.getMergeValues(Ops1, 2); 5012} 5013 5014SDValue 5015X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 5016 SDValue Chain, 5017 SDValue Dst, SDValue Src, 5018 SDValue Size, unsigned Align, 5019 const Value *DstSV, uint64_t DstSVOff) { 5020 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5021 5022 /// If not DWORD aligned or size is more than the threshold, call the library. 5023 /// The libc version is likely to be faster for these cases. It can use the 5024 /// address value and run time information about the CPU. 5025 if ((Align & 3) == 0 || 5026 !ConstantSize || 5027 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 5028 SDValue InFlag(0, 0); 5029 5030 // Check to see if there is a specialized entry-point for memory zeroing. 5031 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 5032 if (const char *bzeroEntry = 5033 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 5034 MVT IntPtr = getPointerTy(); 5035 const Type *IntPtrTy = getTargetData()->getIntPtrType(); 5036 TargetLowering::ArgListTy Args; 5037 TargetLowering::ArgListEntry Entry; 5038 Entry.Node = Dst; 5039 Entry.Ty = IntPtrTy; 5040 Args.push_back(Entry); 5041 Entry.Node = Size; 5042 Args.push_back(Entry); 5043 std::pair<SDValue,SDValue> CallResult = 5044 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 5045 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 5046 Args, DAG); 5047 return CallResult.second; 5048 } 5049 5050 // Otherwise have the target-independent code call memset. 5051 return SDValue(); 5052 } 5053 5054 uint64_t SizeVal = ConstantSize->getValue(); 5055 SDValue InFlag(0, 0); 5056 MVT AVT; 5057 SDValue Count; 5058 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 5059 unsigned BytesLeft = 0; 5060 bool TwoRepStos = false; 5061 if (ValC) { 5062 unsigned ValReg; 5063 uint64_t Val = ValC->getValue() & 255; 5064 5065 // If the value is a constant, then we can potentially use larger sets. 5066 switch (Align & 3) { 5067 case 2: // WORD aligned 5068 AVT = MVT::i16; 5069 ValReg = X86::AX; 5070 Val = (Val << 8) | Val; 5071 break; 5072 case 0: // DWORD aligned 5073 AVT = MVT::i32; 5074 ValReg = X86::EAX; 5075 Val = (Val << 8) | Val; 5076 Val = (Val << 16) | Val; 5077 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 5078 AVT = MVT::i64; 5079 ValReg = X86::RAX; 5080 Val = (Val << 32) | Val; 5081 } 5082 break; 5083 default: // Byte aligned 5084 AVT = MVT::i8; 5085 ValReg = X86::AL; 5086 Count = DAG.getIntPtrConstant(SizeVal); 5087 break; 5088 } 5089 5090 if (AVT.bitsGT(MVT::i8)) { 5091 unsigned UBytes = AVT.getSizeInBits() / 8; 5092 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 5093 BytesLeft = SizeVal % UBytes; 5094 } 5095 5096 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 5097 InFlag); 5098 InFlag = Chain.getValue(1); 5099 } else { 5100 AVT = MVT::i8; 5101 Count = DAG.getIntPtrConstant(SizeVal); 5102 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 5103 InFlag = Chain.getValue(1); 5104 } 5105 5106 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5107 Count, InFlag); 5108 InFlag = Chain.getValue(1); 5109 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5110 Dst, InFlag); 5111 InFlag = Chain.getValue(1); 5112 5113 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5114 SmallVector<SDValue, 8> Ops; 5115 Ops.push_back(Chain); 5116 Ops.push_back(DAG.getValueType(AVT)); 5117 Ops.push_back(InFlag); 5118 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5119 5120 if (TwoRepStos) { 5121 InFlag = Chain.getValue(1); 5122 Count = Size; 5123 MVT CVT = Count.getValueType(); 5124 SDValue Left = DAG.getNode(ISD::AND, CVT, Count, 5125 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 5126 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 5127 Left, InFlag); 5128 InFlag = Chain.getValue(1); 5129 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5130 Ops.clear(); 5131 Ops.push_back(Chain); 5132 Ops.push_back(DAG.getValueType(MVT::i8)); 5133 Ops.push_back(InFlag); 5134 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5135 } else if (BytesLeft) { 5136 // Handle the last 1 - 7 bytes. 5137 unsigned Offset = SizeVal - BytesLeft; 5138 MVT AddrVT = Dst.getValueType(); 5139 MVT SizeVT = Size.getValueType(); 5140 5141 Chain = DAG.getMemset(Chain, 5142 DAG.getNode(ISD::ADD, AddrVT, Dst, 5143 DAG.getConstant(Offset, AddrVT)), 5144 Src, 5145 DAG.getConstant(BytesLeft, SizeVT), 5146 Align, DstSV, DstSVOff + Offset); 5147 } 5148 5149 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 5150 return Chain; 5151} 5152 5153SDValue 5154X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 5155 SDValue Chain, 5156 SDValue Dst, SDValue Src, 5157 SDValue Size, unsigned Align, 5158 bool AlwaysInline, 5159 const Value *DstSV, uint64_t DstSVOff, 5160 const Value *SrcSV, uint64_t SrcSVOff){ 5161 5162 // This requires the copy size to be a constant, preferrably 5163 // within a subtarget-specific limit. 5164 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5165 if (!ConstantSize) 5166 return SDValue(); 5167 uint64_t SizeVal = ConstantSize->getValue(); 5168 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 5169 return SDValue(); 5170 5171 MVT AVT; 5172 unsigned BytesLeft = 0; 5173 if (Align >= 8 && Subtarget->is64Bit()) 5174 AVT = MVT::i64; 5175 else if (Align >= 4) 5176 AVT = MVT::i32; 5177 else if (Align >= 2) 5178 AVT = MVT::i16; 5179 else 5180 AVT = MVT::i8; 5181 5182 unsigned UBytes = AVT.getSizeInBits() / 8; 5183 unsigned CountVal = SizeVal / UBytes; 5184 SDValue Count = DAG.getIntPtrConstant(CountVal); 5185 BytesLeft = SizeVal % UBytes; 5186 5187 SDValue InFlag(0, 0); 5188 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5189 Count, InFlag); 5190 InFlag = Chain.getValue(1); 5191 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5192 Dst, InFlag); 5193 InFlag = Chain.getValue(1); 5194 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 5195 Src, InFlag); 5196 InFlag = Chain.getValue(1); 5197 5198 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5199 SmallVector<SDValue, 8> Ops; 5200 Ops.push_back(Chain); 5201 Ops.push_back(DAG.getValueType(AVT)); 5202 Ops.push_back(InFlag); 5203 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5204 5205 SmallVector<SDValue, 4> Results; 5206 Results.push_back(RepMovs); 5207 if (BytesLeft) { 5208 // Handle the last 1 - 7 bytes. 5209 unsigned Offset = SizeVal - BytesLeft; 5210 MVT DstVT = Dst.getValueType(); 5211 MVT SrcVT = Src.getValueType(); 5212 MVT SizeVT = Size.getValueType(); 5213 Results.push_back(DAG.getMemcpy(Chain, 5214 DAG.getNode(ISD::ADD, DstVT, Dst, 5215 DAG.getConstant(Offset, DstVT)), 5216 DAG.getNode(ISD::ADD, SrcVT, Src, 5217 DAG.getConstant(Offset, SrcVT)), 5218 DAG.getConstant(BytesLeft, SizeVT), 5219 Align, AlwaysInline, 5220 DstSV, DstSVOff + Offset, 5221 SrcSV, SrcSVOff + Offset)); 5222 } 5223 5224 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5225} 5226 5227/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5228SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5229 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5230 SDValue TheChain = N->getOperand(0); 5231 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5232 if (Subtarget->is64Bit()) { 5233 SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5234 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5235 MVT::i64, rax.getValue(2)); 5236 SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5237 DAG.getConstant(32, MVT::i8)); 5238 SDValue Ops[] = { 5239 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5240 }; 5241 5242 return DAG.getMergeValues(Ops, 2).Val; 5243 } 5244 5245 SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5246 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5247 MVT::i32, eax.getValue(2)); 5248 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5249 SDValue Ops[] = { eax, edx }; 5250 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5251 5252 // Use a MERGE_VALUES to return the value and chain. 5253 Ops[1] = edx.getValue(1); 5254 return DAG.getMergeValues(Ops, 2).Val; 5255} 5256 5257SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) { 5258 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5259 5260 if (!Subtarget->is64Bit()) { 5261 // vastart just stores the address of the VarArgsFrameIndex slot into the 5262 // memory location argument. 5263 SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5264 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5265 } 5266 5267 // __va_list_tag: 5268 // gp_offset (0 - 6 * 8) 5269 // fp_offset (48 - 48 + 8 * 16) 5270 // overflow_arg_area (point to parameters coming in memory). 5271 // reg_save_area 5272 SmallVector<SDValue, 8> MemOps; 5273 SDValue FIN = Op.getOperand(1); 5274 // Store gp_offset 5275 SDValue Store = DAG.getStore(Op.getOperand(0), 5276 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5277 FIN, SV, 0); 5278 MemOps.push_back(Store); 5279 5280 // Store fp_offset 5281 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5282 Store = DAG.getStore(Op.getOperand(0), 5283 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5284 FIN, SV, 0); 5285 MemOps.push_back(Store); 5286 5287 // Store ptr to overflow_arg_area 5288 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5289 SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5290 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5291 MemOps.push_back(Store); 5292 5293 // Store ptr to reg_save_area. 5294 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5295 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5296 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5297 MemOps.push_back(Store); 5298 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5299} 5300 5301SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) { 5302 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5303 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5304 SDValue Chain = Op.getOperand(0); 5305 SDValue SrcPtr = Op.getOperand(1); 5306 SDValue SrcSV = Op.getOperand(2); 5307 5308 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5309 abort(); 5310 return SDValue(); 5311} 5312 5313SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) { 5314 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5315 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5316 SDValue Chain = Op.getOperand(0); 5317 SDValue DstPtr = Op.getOperand(1); 5318 SDValue SrcPtr = Op.getOperand(2); 5319 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5320 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5321 5322 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5323 DAG.getIntPtrConstant(24), 8, false, 5324 DstSV, 0, SrcSV, 0); 5325} 5326 5327SDValue 5328X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { 5329 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5330 switch (IntNo) { 5331 default: return SDValue(); // Don't custom lower most intrinsics. 5332 // Comparison intrinsics. 5333 case Intrinsic::x86_sse_comieq_ss: 5334 case Intrinsic::x86_sse_comilt_ss: 5335 case Intrinsic::x86_sse_comile_ss: 5336 case Intrinsic::x86_sse_comigt_ss: 5337 case Intrinsic::x86_sse_comige_ss: 5338 case Intrinsic::x86_sse_comineq_ss: 5339 case Intrinsic::x86_sse_ucomieq_ss: 5340 case Intrinsic::x86_sse_ucomilt_ss: 5341 case Intrinsic::x86_sse_ucomile_ss: 5342 case Intrinsic::x86_sse_ucomigt_ss: 5343 case Intrinsic::x86_sse_ucomige_ss: 5344 case Intrinsic::x86_sse_ucomineq_ss: 5345 case Intrinsic::x86_sse2_comieq_sd: 5346 case Intrinsic::x86_sse2_comilt_sd: 5347 case Intrinsic::x86_sse2_comile_sd: 5348 case Intrinsic::x86_sse2_comigt_sd: 5349 case Intrinsic::x86_sse2_comige_sd: 5350 case Intrinsic::x86_sse2_comineq_sd: 5351 case Intrinsic::x86_sse2_ucomieq_sd: 5352 case Intrinsic::x86_sse2_ucomilt_sd: 5353 case Intrinsic::x86_sse2_ucomile_sd: 5354 case Intrinsic::x86_sse2_ucomigt_sd: 5355 case Intrinsic::x86_sse2_ucomige_sd: 5356 case Intrinsic::x86_sse2_ucomineq_sd: { 5357 unsigned Opc = 0; 5358 ISD::CondCode CC = ISD::SETCC_INVALID; 5359 switch (IntNo) { 5360 default: break; 5361 case Intrinsic::x86_sse_comieq_ss: 5362 case Intrinsic::x86_sse2_comieq_sd: 5363 Opc = X86ISD::COMI; 5364 CC = ISD::SETEQ; 5365 break; 5366 case Intrinsic::x86_sse_comilt_ss: 5367 case Intrinsic::x86_sse2_comilt_sd: 5368 Opc = X86ISD::COMI; 5369 CC = ISD::SETLT; 5370 break; 5371 case Intrinsic::x86_sse_comile_ss: 5372 case Intrinsic::x86_sse2_comile_sd: 5373 Opc = X86ISD::COMI; 5374 CC = ISD::SETLE; 5375 break; 5376 case Intrinsic::x86_sse_comigt_ss: 5377 case Intrinsic::x86_sse2_comigt_sd: 5378 Opc = X86ISD::COMI; 5379 CC = ISD::SETGT; 5380 break; 5381 case Intrinsic::x86_sse_comige_ss: 5382 case Intrinsic::x86_sse2_comige_sd: 5383 Opc = X86ISD::COMI; 5384 CC = ISD::SETGE; 5385 break; 5386 case Intrinsic::x86_sse_comineq_ss: 5387 case Intrinsic::x86_sse2_comineq_sd: 5388 Opc = X86ISD::COMI; 5389 CC = ISD::SETNE; 5390 break; 5391 case Intrinsic::x86_sse_ucomieq_ss: 5392 case Intrinsic::x86_sse2_ucomieq_sd: 5393 Opc = X86ISD::UCOMI; 5394 CC = ISD::SETEQ; 5395 break; 5396 case Intrinsic::x86_sse_ucomilt_ss: 5397 case Intrinsic::x86_sse2_ucomilt_sd: 5398 Opc = X86ISD::UCOMI; 5399 CC = ISD::SETLT; 5400 break; 5401 case Intrinsic::x86_sse_ucomile_ss: 5402 case Intrinsic::x86_sse2_ucomile_sd: 5403 Opc = X86ISD::UCOMI; 5404 CC = ISD::SETLE; 5405 break; 5406 case Intrinsic::x86_sse_ucomigt_ss: 5407 case Intrinsic::x86_sse2_ucomigt_sd: 5408 Opc = X86ISD::UCOMI; 5409 CC = ISD::SETGT; 5410 break; 5411 case Intrinsic::x86_sse_ucomige_ss: 5412 case Intrinsic::x86_sse2_ucomige_sd: 5413 Opc = X86ISD::UCOMI; 5414 CC = ISD::SETGE; 5415 break; 5416 case Intrinsic::x86_sse_ucomineq_ss: 5417 case Intrinsic::x86_sse2_ucomineq_sd: 5418 Opc = X86ISD::UCOMI; 5419 CC = ISD::SETNE; 5420 break; 5421 } 5422 5423 unsigned X86CC; 5424 SDValue LHS = Op.getOperand(1); 5425 SDValue RHS = Op.getOperand(2); 5426 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5427 5428 SDValue Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5429 SDValue SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5430 DAG.getConstant(X86CC, MVT::i8), Cond); 5431 return DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, SetCC); 5432 } 5433 5434 // Fix vector shift instructions where the last operand is a non-immediate 5435 // i32 value. 5436 case Intrinsic::x86_sse2_pslli_w: 5437 case Intrinsic::x86_sse2_pslli_d: 5438 case Intrinsic::x86_sse2_pslli_q: 5439 case Intrinsic::x86_sse2_psrli_w: 5440 case Intrinsic::x86_sse2_psrli_d: 5441 case Intrinsic::x86_sse2_psrli_q: 5442 case Intrinsic::x86_sse2_psrai_w: 5443 case Intrinsic::x86_sse2_psrai_d: 5444 case Intrinsic::x86_mmx_pslli_w: 5445 case Intrinsic::x86_mmx_pslli_d: 5446 case Intrinsic::x86_mmx_pslli_q: 5447 case Intrinsic::x86_mmx_psrli_w: 5448 case Intrinsic::x86_mmx_psrli_d: 5449 case Intrinsic::x86_mmx_psrli_q: 5450 case Intrinsic::x86_mmx_psrai_w: 5451 case Intrinsic::x86_mmx_psrai_d: { 5452 SDValue ShAmt = Op.getOperand(2); 5453 if (isa<ConstantSDNode>(ShAmt)) 5454 return SDValue(); 5455 5456 unsigned NewIntNo = 0; 5457 MVT ShAmtVT = MVT::v4i32; 5458 switch (IntNo) { 5459 case Intrinsic::x86_sse2_pslli_w: 5460 NewIntNo = Intrinsic::x86_sse2_psll_w; 5461 break; 5462 case Intrinsic::x86_sse2_pslli_d: 5463 NewIntNo = Intrinsic::x86_sse2_psll_d; 5464 break; 5465 case Intrinsic::x86_sse2_pslli_q: 5466 NewIntNo = Intrinsic::x86_sse2_psll_q; 5467 break; 5468 case Intrinsic::x86_sse2_psrli_w: 5469 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5470 break; 5471 case Intrinsic::x86_sse2_psrli_d: 5472 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5473 break; 5474 case Intrinsic::x86_sse2_psrli_q: 5475 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5476 break; 5477 case Intrinsic::x86_sse2_psrai_w: 5478 NewIntNo = Intrinsic::x86_sse2_psra_w; 5479 break; 5480 case Intrinsic::x86_sse2_psrai_d: 5481 NewIntNo = Intrinsic::x86_sse2_psra_d; 5482 break; 5483 default: { 5484 ShAmtVT = MVT::v2i32; 5485 switch (IntNo) { 5486 case Intrinsic::x86_mmx_pslli_w: 5487 NewIntNo = Intrinsic::x86_mmx_psll_w; 5488 break; 5489 case Intrinsic::x86_mmx_pslli_d: 5490 NewIntNo = Intrinsic::x86_mmx_psll_d; 5491 break; 5492 case Intrinsic::x86_mmx_pslli_q: 5493 NewIntNo = Intrinsic::x86_mmx_psll_q; 5494 break; 5495 case Intrinsic::x86_mmx_psrli_w: 5496 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5497 break; 5498 case Intrinsic::x86_mmx_psrli_d: 5499 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5500 break; 5501 case Intrinsic::x86_mmx_psrli_q: 5502 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5503 break; 5504 case Intrinsic::x86_mmx_psrai_w: 5505 NewIntNo = Intrinsic::x86_mmx_psra_w; 5506 break; 5507 case Intrinsic::x86_mmx_psrai_d: 5508 NewIntNo = Intrinsic::x86_mmx_psra_d; 5509 break; 5510 default: abort(); // Can't reach here. 5511 } 5512 break; 5513 } 5514 } 5515 MVT VT = Op.getValueType(); 5516 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5517 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5518 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5519 DAG.getConstant(NewIntNo, MVT::i32), 5520 Op.getOperand(1), ShAmt); 5521 } 5522 } 5523} 5524 5525SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) { 5526 // Depths > 0 not supported yet! 5527 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5528 return SDValue(); 5529 5530 // Just load the return address 5531 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5532 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5533} 5534 5535SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) { 5536 // Depths > 0 not supported yet! 5537 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5538 return SDValue(); 5539 5540 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5541 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5542 DAG.getIntPtrConstant(!Subtarget->is64Bit() ? 4 : 8)); 5543} 5544 5545SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 5546 SelectionDAG &DAG) { 5547 // Is not yet supported on x86-64 5548 if (Subtarget->is64Bit()) 5549 return SDValue(); 5550 5551 return DAG.getIntPtrConstant(8); 5552} 5553 5554SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) 5555{ 5556 assert(!Subtarget->is64Bit() && 5557 "Lowering of eh_return builtin is not supported yet on x86-64"); 5558 5559 MachineFunction &MF = DAG.getMachineFunction(); 5560 SDValue Chain = Op.getOperand(0); 5561 SDValue Offset = Op.getOperand(1); 5562 SDValue Handler = Op.getOperand(2); 5563 5564 SDValue Frame = DAG.getRegister(RegInfo->getFrameRegister(MF), 5565 getPointerTy()); 5566 5567 SDValue StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5568 DAG.getIntPtrConstant(-4UL)); 5569 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5570 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5571 Chain = DAG.getCopyToReg(Chain, X86::ECX, StoreAddr); 5572 MF.getRegInfo().addLiveOut(X86::ECX); 5573 5574 return DAG.getNode(X86ISD::EH_RETURN, MVT::Other, 5575 Chain, DAG.getRegister(X86::ECX, getPointerTy())); 5576} 5577 5578SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, 5579 SelectionDAG &DAG) { 5580 SDValue Root = Op.getOperand(0); 5581 SDValue Trmp = Op.getOperand(1); // trampoline 5582 SDValue FPtr = Op.getOperand(2); // nested function 5583 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 5584 5585 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5586 5587 const X86InstrInfo *TII = 5588 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5589 5590 if (Subtarget->is64Bit()) { 5591 SDValue OutChains[6]; 5592 5593 // Large code-model. 5594 5595 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5596 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5597 5598 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5599 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5600 5601 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5602 5603 // Load the pointer to the nested function into R11. 5604 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5605 SDValue Addr = Trmp; 5606 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5607 TrmpAddr, 0); 5608 5609 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5610 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5611 5612 // Load the 'nest' parameter value into R10. 5613 // R10 is specified in X86CallingConv.td 5614 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5615 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5616 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5617 TrmpAddr, 10); 5618 5619 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5620 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5621 5622 // Jump to the nested function. 5623 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5624 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5625 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5626 TrmpAddr, 20); 5627 5628 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5629 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5630 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5631 TrmpAddr, 22); 5632 5633 SDValue Ops[] = 5634 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5635 return DAG.getMergeValues(Ops, 2); 5636 } else { 5637 const Function *Func = 5638 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5639 unsigned CC = Func->getCallingConv(); 5640 unsigned NestReg; 5641 5642 switch (CC) { 5643 default: 5644 assert(0 && "Unsupported calling convention"); 5645 case CallingConv::C: 5646 case CallingConv::X86_StdCall: { 5647 // Pass 'nest' parameter in ECX. 5648 // Must be kept in sync with X86CallingConv.td 5649 NestReg = X86::ECX; 5650 5651 // Check that ECX wasn't needed by an 'inreg' parameter. 5652 const FunctionType *FTy = Func->getFunctionType(); 5653 const PAListPtr &Attrs = Func->getParamAttrs(); 5654 5655 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5656 unsigned InRegCount = 0; 5657 unsigned Idx = 1; 5658 5659 for (FunctionType::param_iterator I = FTy->param_begin(), 5660 E = FTy->param_end(); I != E; ++I, ++Idx) 5661 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5662 // FIXME: should only count parameters that are lowered to integers. 5663 InRegCount += (getTargetData()->getTypeSizeInBits(*I) + 31) / 32; 5664 5665 if (InRegCount > 2) { 5666 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5667 abort(); 5668 } 5669 } 5670 break; 5671 } 5672 case CallingConv::X86_FastCall: 5673 // Pass 'nest' parameter in EAX. 5674 // Must be kept in sync with X86CallingConv.td 5675 NestReg = X86::EAX; 5676 break; 5677 } 5678 5679 SDValue OutChains[4]; 5680 SDValue Addr, Disp; 5681 5682 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5683 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5684 5685 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5686 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5687 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5688 Trmp, TrmpAddr, 0); 5689 5690 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5691 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5692 5693 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5694 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5695 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5696 TrmpAddr, 5, false, 1); 5697 5698 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5699 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5700 5701 SDValue Ops[] = 5702 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5703 return DAG.getMergeValues(Ops, 2); 5704 } 5705} 5706 5707SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) { 5708 /* 5709 The rounding mode is in bits 11:10 of FPSR, and has the following 5710 settings: 5711 00 Round to nearest 5712 01 Round to -inf 5713 10 Round to +inf 5714 11 Round to 0 5715 5716 FLT_ROUNDS, on the other hand, expects the following: 5717 -1 Undefined 5718 0 Round to 0 5719 1 Round to nearest 5720 2 Round to +inf 5721 3 Round to -inf 5722 5723 To perform the conversion, we do: 5724 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5725 */ 5726 5727 MachineFunction &MF = DAG.getMachineFunction(); 5728 const TargetMachine &TM = MF.getTarget(); 5729 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5730 unsigned StackAlignment = TFI.getStackAlignment(); 5731 MVT VT = Op.getValueType(); 5732 5733 // Save FP Control Word to stack slot 5734 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5735 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5736 5737 SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5738 DAG.getEntryNode(), StackSlot); 5739 5740 // Load FP Control Word from stack slot 5741 SDValue CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5742 5743 // Transform as necessary 5744 SDValue CWD1 = 5745 DAG.getNode(ISD::SRL, MVT::i16, 5746 DAG.getNode(ISD::AND, MVT::i16, 5747 CWD, DAG.getConstant(0x800, MVT::i16)), 5748 DAG.getConstant(11, MVT::i8)); 5749 SDValue CWD2 = 5750 DAG.getNode(ISD::SRL, MVT::i16, 5751 DAG.getNode(ISD::AND, MVT::i16, 5752 CWD, DAG.getConstant(0x400, MVT::i16)), 5753 DAG.getConstant(9, MVT::i8)); 5754 5755 SDValue RetVal = 5756 DAG.getNode(ISD::AND, MVT::i16, 5757 DAG.getNode(ISD::ADD, MVT::i16, 5758 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5759 DAG.getConstant(1, MVT::i16)), 5760 DAG.getConstant(3, MVT::i16)); 5761 5762 5763 return DAG.getNode((VT.getSizeInBits() < 16 ? 5764 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5765} 5766 5767SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) { 5768 MVT VT = Op.getValueType(); 5769 MVT OpVT = VT; 5770 unsigned NumBits = VT.getSizeInBits(); 5771 5772 Op = Op.getOperand(0); 5773 if (VT == MVT::i8) { 5774 // Zero extend to i32 since there is not an i8 bsr. 5775 OpVT = MVT::i32; 5776 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5777 } 5778 5779 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5780 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5781 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5782 5783 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5784 SmallVector<SDValue, 4> Ops; 5785 Ops.push_back(Op); 5786 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5787 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5788 Ops.push_back(Op.getValue(1)); 5789 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5790 5791 // Finally xor with NumBits-1. 5792 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5793 5794 if (VT == MVT::i8) 5795 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5796 return Op; 5797} 5798 5799SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) { 5800 MVT VT = Op.getValueType(); 5801 MVT OpVT = VT; 5802 unsigned NumBits = VT.getSizeInBits(); 5803 5804 Op = Op.getOperand(0); 5805 if (VT == MVT::i8) { 5806 OpVT = MVT::i32; 5807 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5808 } 5809 5810 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5811 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5812 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5813 5814 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5815 SmallVector<SDValue, 4> Ops; 5816 Ops.push_back(Op); 5817 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5818 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5819 Ops.push_back(Op.getValue(1)); 5820 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5821 5822 if (VT == MVT::i8) 5823 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5824 return Op; 5825} 5826 5827SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) { 5828 MVT T = Op.getValueType(); 5829 unsigned Reg = 0; 5830 unsigned size = 0; 5831 switch(T.getSimpleVT()) { 5832 default: 5833 assert(false && "Invalid value type!"); 5834 case MVT::i8: Reg = X86::AL; size = 1; break; 5835 case MVT::i16: Reg = X86::AX; size = 2; break; 5836 case MVT::i32: Reg = X86::EAX; size = 4; break; 5837 case MVT::i64: 5838 if (Subtarget->is64Bit()) { 5839 Reg = X86::RAX; size = 8; 5840 } else //Should go away when LowerType stuff lands 5841 return SDValue(ExpandATOMIC_CMP_SWAP(Op.Val, DAG), 0); 5842 break; 5843 }; 5844 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5845 Op.getOperand(3), SDValue()); 5846 SDValue Ops[] = { cpIn.getValue(0), 5847 Op.getOperand(1), 5848 Op.getOperand(2), 5849 DAG.getTargetConstant(size, MVT::i8), 5850 cpIn.getValue(1) }; 5851 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5852 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5853 SDValue cpOut = 5854 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5855 return cpOut; 5856} 5857 5858SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, SelectionDAG &DAG) { 5859 MVT T = Op->getValueType(0); 5860 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5861 SDValue cpInL, cpInH; 5862 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5863 DAG.getConstant(0, MVT::i32)); 5864 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5865 DAG.getConstant(1, MVT::i32)); 5866 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5867 cpInL, SDValue()); 5868 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5869 cpInH, cpInL.getValue(1)); 5870 SDValue swapInL, swapInH; 5871 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5872 DAG.getConstant(0, MVT::i32)); 5873 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5874 DAG.getConstant(1, MVT::i32)); 5875 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5876 swapInL, cpInH.getValue(1)); 5877 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5878 swapInH, swapInL.getValue(1)); 5879 SDValue Ops[] = { swapInH.getValue(0), 5880 Op->getOperand(1), 5881 swapInH.getValue(1)}; 5882 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5883 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5884 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5885 Result.getValue(1)); 5886 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5887 cpOutL.getValue(2)); 5888 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5889 SDValue ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5890 SDValue Vals[2] = { ResultVal, cpOutH.getValue(1) }; 5891 return DAG.getMergeValues(Vals, 2).Val; 5892} 5893 5894SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, SelectionDAG &DAG) { 5895 MVT T = Op->getValueType(0); 5896 SDValue negOp = DAG.getNode(ISD::SUB, T, 5897 DAG.getConstant(0, T), Op->getOperand(2)); 5898 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, Op->getOperand(0), 5899 Op->getOperand(1), negOp, 5900 cast<AtomicSDNode>(Op)->getSrcValue(), 5901 cast<AtomicSDNode>(Op)->getAlignment()).Val; 5902} 5903 5904/// LowerOperation - Provide custom lowering hooks for some operations. 5905/// 5906SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { 5907 switch (Op.getOpcode()) { 5908 default: assert(0 && "Should not custom lower this!"); 5909 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 5910 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5911 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5912 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5913 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5914 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5915 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5916 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5917 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5918 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5919 case ISD::SHL_PARTS: 5920 case ISD::SRA_PARTS: 5921 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5922 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5923 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5924 case ISD::FABS: return LowerFABS(Op, DAG); 5925 case ISD::FNEG: return LowerFNEG(Op, DAG); 5926 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5927 case ISD::SETCC: return LowerSETCC(Op, DAG); 5928 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 5929 case ISD::SELECT: return LowerSELECT(Op, DAG); 5930 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5931 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5932 case ISD::CALL: return LowerCALL(Op, DAG); 5933 case ISD::RET: return LowerRET(Op, DAG); 5934 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 5935 case ISD::VASTART: return LowerVASTART(Op, DAG); 5936 case ISD::VAARG: return LowerVAARG(Op, DAG); 5937 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 5938 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 5939 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 5940 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 5941 case ISD::FRAME_TO_ARGS_OFFSET: 5942 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 5943 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 5944 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 5945 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 5946 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 5947 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 5948 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 5949 5950 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 5951 case ISD::READCYCLECOUNTER: 5952 return SDValue(ExpandREADCYCLECOUNTER(Op.Val, DAG), 0); 5953 } 5954} 5955 5956/// ReplaceNodeResults - Replace a node with an illegal result type 5957/// with a new node built out of custom code. 5958SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) { 5959 switch (N->getOpcode()) { 5960 default: assert(0 && "Should not custom lower this!"); 5961 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 5962 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 5963 case ISD::ATOMIC_CMP_SWAP: return ExpandATOMIC_CMP_SWAP(N, DAG); 5964 case ISD::ATOMIC_LOAD_SUB: return ExpandATOMIC_LOAD_SUB(N,DAG); 5965 } 5966} 5967 5968const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 5969 switch (Opcode) { 5970 default: return NULL; 5971 case X86ISD::BSF: return "X86ISD::BSF"; 5972 case X86ISD::BSR: return "X86ISD::BSR"; 5973 case X86ISD::SHLD: return "X86ISD::SHLD"; 5974 case X86ISD::SHRD: return "X86ISD::SHRD"; 5975 case X86ISD::FAND: return "X86ISD::FAND"; 5976 case X86ISD::FOR: return "X86ISD::FOR"; 5977 case X86ISD::FXOR: return "X86ISD::FXOR"; 5978 case X86ISD::FSRL: return "X86ISD::FSRL"; 5979 case X86ISD::FILD: return "X86ISD::FILD"; 5980 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 5981 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 5982 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 5983 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 5984 case X86ISD::FLD: return "X86ISD::FLD"; 5985 case X86ISD::FST: return "X86ISD::FST"; 5986 case X86ISD::CALL: return "X86ISD::CALL"; 5987 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 5988 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 5989 case X86ISD::CMP: return "X86ISD::CMP"; 5990 case X86ISD::COMI: return "X86ISD::COMI"; 5991 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 5992 case X86ISD::SETCC: return "X86ISD::SETCC"; 5993 case X86ISD::CMOV: return "X86ISD::CMOV"; 5994 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 5995 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 5996 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 5997 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 5998 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 5999 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 6000 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 6001 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 6002 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 6003 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 6004 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 6005 case X86ISD::FMAX: return "X86ISD::FMAX"; 6006 case X86ISD::FMIN: return "X86ISD::FMIN"; 6007 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 6008 case X86ISD::FRCP: return "X86ISD::FRCP"; 6009 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 6010 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 6011 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 6012 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 6013 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 6014 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 6015 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 6016 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 6017 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 6018 case X86ISD::VSHL: return "X86ISD::VSHL"; 6019 case X86ISD::VSRL: return "X86ISD::VSRL"; 6020 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 6021 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 6022 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 6023 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 6024 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 6025 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 6026 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 6027 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 6028 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 6029 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 6030 } 6031} 6032 6033// isLegalAddressingMode - Return true if the addressing mode represented 6034// by AM is legal for this target, for a load/store of the specified type. 6035bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 6036 const Type *Ty) const { 6037 // X86 supports extremely general addressing modes. 6038 6039 // X86 allows a sign-extended 32-bit immediate field as a displacement. 6040 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 6041 return false; 6042 6043 if (AM.BaseGV) { 6044 // We can only fold this if we don't need an extra load. 6045 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 6046 return false; 6047 6048 // X86-64 only supports addr of globals in small code model. 6049 if (Subtarget->is64Bit()) { 6050 if (getTargetMachine().getCodeModel() != CodeModel::Small) 6051 return false; 6052 // If lower 4G is not available, then we must use rip-relative addressing. 6053 if (AM.BaseOffs || AM.Scale > 1) 6054 return false; 6055 } 6056 } 6057 6058 switch (AM.Scale) { 6059 case 0: 6060 case 1: 6061 case 2: 6062 case 4: 6063 case 8: 6064 // These scales always work. 6065 break; 6066 case 3: 6067 case 5: 6068 case 9: 6069 // These scales are formed with basereg+scalereg. Only accept if there is 6070 // no basereg yet. 6071 if (AM.HasBaseReg) 6072 return false; 6073 break; 6074 default: // Other stuff never works. 6075 return false; 6076 } 6077 6078 return true; 6079} 6080 6081 6082bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 6083 if (!Ty1->isInteger() || !Ty2->isInteger()) 6084 return false; 6085 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6086 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6087 if (NumBits1 <= NumBits2) 6088 return false; 6089 return Subtarget->is64Bit() || NumBits1 < 64; 6090} 6091 6092bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 6093 if (!VT1.isInteger() || !VT2.isInteger()) 6094 return false; 6095 unsigned NumBits1 = VT1.getSizeInBits(); 6096 unsigned NumBits2 = VT2.getSizeInBits(); 6097 if (NumBits1 <= NumBits2) 6098 return false; 6099 return Subtarget->is64Bit() || NumBits1 < 64; 6100} 6101 6102/// isShuffleMaskLegal - Targets can use this to indicate that they only 6103/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6104/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6105/// are assumed to be legal. 6106bool 6107X86TargetLowering::isShuffleMaskLegal(SDValue Mask, MVT VT) const { 6108 // Only do shuffles on 128-bit vector types for now. 6109 if (VT.getSizeInBits() == 64) return false; 6110 return (Mask.Val->getNumOperands() <= 4 || 6111 isIdentityMask(Mask.Val) || 6112 isIdentityMask(Mask.Val, true) || 6113 isSplatMask(Mask.Val) || 6114 isPSHUFHW_PSHUFLWMask(Mask.Val) || 6115 X86::isUNPCKLMask(Mask.Val) || 6116 X86::isUNPCKHMask(Mask.Val) || 6117 X86::isUNPCKL_v_undef_Mask(Mask.Val) || 6118 X86::isUNPCKH_v_undef_Mask(Mask.Val)); 6119} 6120 6121bool 6122X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, 6123 MVT EVT, SelectionDAG &DAG) const { 6124 unsigned NumElts = BVOps.size(); 6125 // Only do shuffles on 128-bit vector types for now. 6126 if (EVT.getSizeInBits() * NumElts == 64) return false; 6127 if (NumElts == 2) return true; 6128 if (NumElts == 4) { 6129 return (isMOVLMask(&BVOps[0], 4) || 6130 isCommutedMOVL(&BVOps[0], 4, true) || 6131 isSHUFPMask(&BVOps[0], 4) || 6132 isCommutedSHUFP(&BVOps[0], 4)); 6133 } 6134 return false; 6135} 6136 6137//===----------------------------------------------------------------------===// 6138// X86 Scheduler Hooks 6139//===----------------------------------------------------------------------===// 6140 6141// private utility function 6142MachineBasicBlock * 6143X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 6144 MachineBasicBlock *MBB, 6145 unsigned regOpc, 6146 unsigned immOpc, 6147 unsigned LoadOpc, 6148 unsigned CXchgOpc, 6149 unsigned copyOpc, 6150 unsigned notOpc, 6151 unsigned EAXreg, 6152 TargetRegisterClass *RC, 6153 bool invSrc) { 6154 // For the atomic bitwise operator, we generate 6155 // thisMBB: 6156 // newMBB: 6157 // ld t1 = [bitinstr.addr] 6158 // op t2 = t1, [bitinstr.val] 6159 // mov EAX = t1 6160 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6161 // bz newMBB 6162 // fallthrough -->nextMBB 6163 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6164 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6165 MachineFunction::iterator MBBIter = MBB; 6166 ++MBBIter; 6167 6168 /// First build the CFG 6169 MachineFunction *F = MBB->getParent(); 6170 MachineBasicBlock *thisMBB = MBB; 6171 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6172 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6173 F->insert(MBBIter, newMBB); 6174 F->insert(MBBIter, nextMBB); 6175 6176 // Move all successors to thisMBB to nextMBB 6177 nextMBB->transferSuccessors(thisMBB); 6178 6179 // Update thisMBB to fall through to newMBB 6180 thisMBB->addSuccessor(newMBB); 6181 6182 // newMBB jumps to itself and fall through to nextMBB 6183 newMBB->addSuccessor(nextMBB); 6184 newMBB->addSuccessor(newMBB); 6185 6186 // Insert instructions into newMBB based on incoming instruction 6187 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 6188 MachineOperand& destOper = bInstr->getOperand(0); 6189 MachineOperand* argOpers[6]; 6190 int numArgs = bInstr->getNumOperands() - 1; 6191 for (int i=0; i < numArgs; ++i) 6192 argOpers[i] = &bInstr->getOperand(i+1); 6193 6194 // x86 address has 4 operands: base, index, scale, and displacement 6195 int lastAddrIndx = 3; // [0,3] 6196 int valArgIndx = 4; 6197 6198 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 6199 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(LoadOpc), t1); 6200 for (int i=0; i <= lastAddrIndx; ++i) 6201 (*MIB).addOperand(*argOpers[i]); 6202 6203 unsigned tt = F->getRegInfo().createVirtualRegister(RC); 6204 if (invSrc) { 6205 MIB = BuildMI(newMBB, TII->get(notOpc), tt).addReg(t1); 6206 } 6207 else 6208 tt = t1; 6209 6210 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 6211 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6212 && "invalid operand"); 6213 if (argOpers[valArgIndx]->isReg()) 6214 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6215 else 6216 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6217 MIB.addReg(tt); 6218 (*MIB).addOperand(*argOpers[valArgIndx]); 6219 6220 MIB = BuildMI(newMBB, TII->get(copyOpc), EAXreg); 6221 MIB.addReg(t1); 6222 6223 MIB = BuildMI(newMBB, TII->get(CXchgOpc)); 6224 for (int i=0; i <= lastAddrIndx; ++i) 6225 (*MIB).addOperand(*argOpers[i]); 6226 MIB.addReg(t2); 6227 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6228 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 6229 6230 MIB = BuildMI(newMBB, TII->get(copyOpc), destOper.getReg()); 6231 MIB.addReg(EAXreg); 6232 6233 // insert branch 6234 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6235 6236 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 6237 return nextMBB; 6238} 6239 6240// private utility function 6241MachineBasicBlock * 6242X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6243 MachineBasicBlock *MBB, 6244 unsigned cmovOpc) { 6245 // For the atomic min/max operator, we generate 6246 // thisMBB: 6247 // newMBB: 6248 // ld t1 = [min/max.addr] 6249 // mov t2 = [min/max.val] 6250 // cmp t1, t2 6251 // cmov[cond] t2 = t1 6252 // mov EAX = t1 6253 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6254 // bz newMBB 6255 // fallthrough -->nextMBB 6256 // 6257 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6258 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6259 MachineFunction::iterator MBBIter = MBB; 6260 ++MBBIter; 6261 6262 /// First build the CFG 6263 MachineFunction *F = MBB->getParent(); 6264 MachineBasicBlock *thisMBB = MBB; 6265 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6266 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6267 F->insert(MBBIter, newMBB); 6268 F->insert(MBBIter, nextMBB); 6269 6270 // Move all successors to thisMBB to nextMBB 6271 nextMBB->transferSuccessors(thisMBB); 6272 6273 // Update thisMBB to fall through to newMBB 6274 thisMBB->addSuccessor(newMBB); 6275 6276 // newMBB jumps to newMBB and fall through to nextMBB 6277 newMBB->addSuccessor(nextMBB); 6278 newMBB->addSuccessor(newMBB); 6279 6280 // Insert instructions into newMBB based on incoming instruction 6281 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6282 MachineOperand& destOper = mInstr->getOperand(0); 6283 MachineOperand* argOpers[6]; 6284 int numArgs = mInstr->getNumOperands() - 1; 6285 for (int i=0; i < numArgs; ++i) 6286 argOpers[i] = &mInstr->getOperand(i+1); 6287 6288 // x86 address has 4 operands: base, index, scale, and displacement 6289 int lastAddrIndx = 3; // [0,3] 6290 int valArgIndx = 4; 6291 6292 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6293 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6294 for (int i=0; i <= lastAddrIndx; ++i) 6295 (*MIB).addOperand(*argOpers[i]); 6296 6297 // We only support register and immediate values 6298 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6299 && "invalid operand"); 6300 6301 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6302 if (argOpers[valArgIndx]->isReg()) 6303 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6304 else 6305 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6306 (*MIB).addOperand(*argOpers[valArgIndx]); 6307 6308 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6309 MIB.addReg(t1); 6310 6311 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6312 MIB.addReg(t1); 6313 MIB.addReg(t2); 6314 6315 // Generate movc 6316 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6317 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6318 MIB.addReg(t2); 6319 MIB.addReg(t1); 6320 6321 // Cmp and exchange if none has modified the memory location 6322 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6323 for (int i=0; i <= lastAddrIndx; ++i) 6324 (*MIB).addOperand(*argOpers[i]); 6325 MIB.addReg(t3); 6326 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6327 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 6328 6329 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6330 MIB.addReg(X86::EAX); 6331 6332 // insert branch 6333 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6334 6335 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 6336 return nextMBB; 6337} 6338 6339 6340MachineBasicBlock * 6341X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6342 MachineBasicBlock *BB) { 6343 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6344 switch (MI->getOpcode()) { 6345 default: assert(false && "Unexpected instr type to insert"); 6346 case X86::CMOV_FR32: 6347 case X86::CMOV_FR64: 6348 case X86::CMOV_V4F32: 6349 case X86::CMOV_V2F64: 6350 case X86::CMOV_V2I64: { 6351 // To "insert" a SELECT_CC instruction, we actually have to insert the 6352 // diamond control-flow pattern. The incoming instruction knows the 6353 // destination vreg to set, the condition code register to branch on, the 6354 // true/false values to select between, and a branch opcode to use. 6355 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6356 MachineFunction::iterator It = BB; 6357 ++It; 6358 6359 // thisMBB: 6360 // ... 6361 // TrueVal = ... 6362 // cmpTY ccX, r1, r2 6363 // bCC copy1MBB 6364 // fallthrough --> copy0MBB 6365 MachineBasicBlock *thisMBB = BB; 6366 MachineFunction *F = BB->getParent(); 6367 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6368 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6369 unsigned Opc = 6370 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6371 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6372 F->insert(It, copy0MBB); 6373 F->insert(It, sinkMBB); 6374 // Update machine-CFG edges by transferring all successors of the current 6375 // block to the new block which will contain the Phi node for the select. 6376 sinkMBB->transferSuccessors(BB); 6377 6378 // Add the true and fallthrough blocks as its successors. 6379 BB->addSuccessor(copy0MBB); 6380 BB->addSuccessor(sinkMBB); 6381 6382 // copy0MBB: 6383 // %FalseValue = ... 6384 // # fallthrough to sinkMBB 6385 BB = copy0MBB; 6386 6387 // Update machine-CFG edges 6388 BB->addSuccessor(sinkMBB); 6389 6390 // sinkMBB: 6391 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6392 // ... 6393 BB = sinkMBB; 6394 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6395 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6396 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6397 6398 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6399 return BB; 6400 } 6401 6402 case X86::FP32_TO_INT16_IN_MEM: 6403 case X86::FP32_TO_INT32_IN_MEM: 6404 case X86::FP32_TO_INT64_IN_MEM: 6405 case X86::FP64_TO_INT16_IN_MEM: 6406 case X86::FP64_TO_INT32_IN_MEM: 6407 case X86::FP64_TO_INT64_IN_MEM: 6408 case X86::FP80_TO_INT16_IN_MEM: 6409 case X86::FP80_TO_INT32_IN_MEM: 6410 case X86::FP80_TO_INT64_IN_MEM: { 6411 // Change the floating point control register to use "round towards zero" 6412 // mode when truncating to an integer value. 6413 MachineFunction *F = BB->getParent(); 6414 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6415 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6416 6417 // Load the old value of the high byte of the control word... 6418 unsigned OldCW = 6419 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6420 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6421 6422 // Set the high part to be round to zero... 6423 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6424 .addImm(0xC7F); 6425 6426 // Reload the modified control word now... 6427 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6428 6429 // Restore the memory image of control word to original value 6430 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6431 .addReg(OldCW); 6432 6433 // Get the X86 opcode to use. 6434 unsigned Opc; 6435 switch (MI->getOpcode()) { 6436 default: assert(0 && "illegal opcode!"); 6437 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6438 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6439 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6440 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6441 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6442 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6443 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6444 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6445 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6446 } 6447 6448 X86AddressMode AM; 6449 MachineOperand &Op = MI->getOperand(0); 6450 if (Op.isRegister()) { 6451 AM.BaseType = X86AddressMode::RegBase; 6452 AM.Base.Reg = Op.getReg(); 6453 } else { 6454 AM.BaseType = X86AddressMode::FrameIndexBase; 6455 AM.Base.FrameIndex = Op.getIndex(); 6456 } 6457 Op = MI->getOperand(1); 6458 if (Op.isImmediate()) 6459 AM.Scale = Op.getImm(); 6460 Op = MI->getOperand(2); 6461 if (Op.isImmediate()) 6462 AM.IndexReg = Op.getImm(); 6463 Op = MI->getOperand(3); 6464 if (Op.isGlobalAddress()) { 6465 AM.GV = Op.getGlobal(); 6466 } else { 6467 AM.Disp = Op.getImm(); 6468 } 6469 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6470 .addReg(MI->getOperand(4).getReg()); 6471 6472 // Reload the original control word now. 6473 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6474 6475 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6476 return BB; 6477 } 6478 case X86::ATOMAND32: 6479 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6480 X86::AND32ri, X86::MOV32rm, 6481 X86::LCMPXCHG32, X86::MOV32rr, 6482 X86::NOT32r, X86::EAX, 6483 X86::GR32RegisterClass); 6484 case X86::ATOMOR32: 6485 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6486 X86::OR32ri, X86::MOV32rm, 6487 X86::LCMPXCHG32, X86::MOV32rr, 6488 X86::NOT32r, X86::EAX, 6489 X86::GR32RegisterClass); 6490 case X86::ATOMXOR32: 6491 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6492 X86::XOR32ri, X86::MOV32rm, 6493 X86::LCMPXCHG32, X86::MOV32rr, 6494 X86::NOT32r, X86::EAX, 6495 X86::GR32RegisterClass); 6496 case X86::ATOMNAND32: 6497 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6498 X86::AND32ri, X86::MOV32rm, 6499 X86::LCMPXCHG32, X86::MOV32rr, 6500 X86::NOT32r, X86::EAX, 6501 X86::GR32RegisterClass, true); 6502 case X86::ATOMMIN32: 6503 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6504 case X86::ATOMMAX32: 6505 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6506 case X86::ATOMUMIN32: 6507 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6508 case X86::ATOMUMAX32: 6509 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6510 6511 case X86::ATOMAND16: 6512 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6513 X86::AND16ri, X86::MOV16rm, 6514 X86::LCMPXCHG16, X86::MOV16rr, 6515 X86::NOT16r, X86::AX, 6516 X86::GR16RegisterClass); 6517 case X86::ATOMOR16: 6518 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 6519 X86::OR16ri, X86::MOV16rm, 6520 X86::LCMPXCHG16, X86::MOV16rr, 6521 X86::NOT16r, X86::AX, 6522 X86::GR16RegisterClass); 6523 case X86::ATOMXOR16: 6524 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 6525 X86::XOR16ri, X86::MOV16rm, 6526 X86::LCMPXCHG16, X86::MOV16rr, 6527 X86::NOT16r, X86::AX, 6528 X86::GR16RegisterClass); 6529 case X86::ATOMNAND16: 6530 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6531 X86::AND16ri, X86::MOV16rm, 6532 X86::LCMPXCHG16, X86::MOV16rr, 6533 X86::NOT16r, X86::AX, 6534 X86::GR16RegisterClass, true); 6535 case X86::ATOMMIN16: 6536 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 6537 case X86::ATOMMAX16: 6538 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 6539 case X86::ATOMUMIN16: 6540 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 6541 case X86::ATOMUMAX16: 6542 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 6543 6544 case X86::ATOMAND8: 6545 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6546 X86::AND8ri, X86::MOV8rm, 6547 X86::LCMPXCHG8, X86::MOV8rr, 6548 X86::NOT8r, X86::AL, 6549 X86::GR8RegisterClass); 6550 case X86::ATOMOR8: 6551 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 6552 X86::OR8ri, X86::MOV8rm, 6553 X86::LCMPXCHG8, X86::MOV8rr, 6554 X86::NOT8r, X86::AL, 6555 X86::GR8RegisterClass); 6556 case X86::ATOMXOR8: 6557 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 6558 X86::XOR8ri, X86::MOV8rm, 6559 X86::LCMPXCHG8, X86::MOV8rr, 6560 X86::NOT8r, X86::AL, 6561 X86::GR8RegisterClass); 6562 case X86::ATOMNAND8: 6563 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6564 X86::AND8ri, X86::MOV8rm, 6565 X86::LCMPXCHG8, X86::MOV8rr, 6566 X86::NOT8r, X86::AL, 6567 X86::GR8RegisterClass, true); 6568 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 6569 } 6570} 6571 6572//===----------------------------------------------------------------------===// 6573// X86 Optimization Hooks 6574//===----------------------------------------------------------------------===// 6575 6576void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 6577 const APInt &Mask, 6578 APInt &KnownZero, 6579 APInt &KnownOne, 6580 const SelectionDAG &DAG, 6581 unsigned Depth) const { 6582 unsigned Opc = Op.getOpcode(); 6583 assert((Opc >= ISD::BUILTIN_OP_END || 6584 Opc == ISD::INTRINSIC_WO_CHAIN || 6585 Opc == ISD::INTRINSIC_W_CHAIN || 6586 Opc == ISD::INTRINSIC_VOID) && 6587 "Should use MaskedValueIsZero if you don't know whether Op" 6588 " is a target node!"); 6589 6590 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6591 switch (Opc) { 6592 default: break; 6593 case X86ISD::SETCC: 6594 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6595 Mask.getBitWidth() - 1); 6596 break; 6597 } 6598} 6599 6600/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6601/// node is a GlobalAddress + offset. 6602bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6603 GlobalValue* &GA, int64_t &Offset) const{ 6604 if (N->getOpcode() == X86ISD::Wrapper) { 6605 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6606 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6607 return true; 6608 } 6609 } 6610 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6611} 6612 6613static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6614 const TargetLowering &TLI) { 6615 GlobalValue *GV; 6616 int64_t Offset = 0; 6617 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6618 return (GV->getAlignment() >= N && (Offset % N) == 0); 6619 // DAG combine handles the stack object case. 6620 return false; 6621} 6622 6623static bool EltsFromConsecutiveLoads(SDNode *N, SDValue PermMask, 6624 unsigned NumElems, MVT EVT, 6625 SDNode *&Base, 6626 SelectionDAG &DAG, MachineFrameInfo *MFI, 6627 const TargetLowering &TLI) { 6628 Base = NULL; 6629 for (unsigned i = 0; i < NumElems; ++i) { 6630 SDValue Idx = PermMask.getOperand(i); 6631 if (Idx.getOpcode() == ISD::UNDEF) { 6632 if (!Base) 6633 return false; 6634 continue; 6635 } 6636 6637 SDValue Elt = DAG.getShuffleScalarElt(N, i); 6638 if (!Elt.Val || 6639 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.Val))) 6640 return false; 6641 if (!Base) { 6642 Base = Elt.Val; 6643 if (Base->getOpcode() == ISD::UNDEF) 6644 return false; 6645 continue; 6646 } 6647 if (Elt.getOpcode() == ISD::UNDEF) 6648 continue; 6649 6650 if (!TLI.isConsecutiveLoad(Elt.Val, Base, 6651 EVT.getSizeInBits()/8, i, MFI)) 6652 return false; 6653 } 6654 return true; 6655} 6656 6657/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6658/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6659/// if the load addresses are consecutive, non-overlapping, and in the right 6660/// order. 6661static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6662 const TargetLowering &TLI) { 6663 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6664 MVT VT = N->getValueType(0); 6665 MVT EVT = VT.getVectorElementType(); 6666 SDValue PermMask = N->getOperand(2); 6667 unsigned NumElems = PermMask.getNumOperands(); 6668 SDNode *Base = NULL; 6669 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6670 DAG, MFI, TLI)) 6671 return SDValue(); 6672 6673 LoadSDNode *LD = cast<LoadSDNode>(Base); 6674 if (isBaseAlignmentOfN(16, Base->getOperand(1).Val, TLI)) 6675 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6676 LD->getSrcValueOffset(), LD->isVolatile()); 6677 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6678 LD->getSrcValueOffset(), LD->isVolatile(), 6679 LD->getAlignment()); 6680} 6681 6682/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6683static SDValue PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6684 const X86Subtarget *Subtarget, 6685 const TargetLowering &TLI) { 6686 unsigned NumOps = N->getNumOperands(); 6687 6688 // Ignore single operand BUILD_VECTOR. 6689 if (NumOps == 1) 6690 return SDValue(); 6691 6692 MVT VT = N->getValueType(0); 6693 MVT EVT = VT.getVectorElementType(); 6694 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6695 // We are looking for load i64 and zero extend. We want to transform 6696 // it before legalizer has a chance to expand it. Also look for i64 6697 // BUILD_PAIR bit casted to f64. 6698 return SDValue(); 6699 // This must be an insertion into a zero vector. 6700 SDValue HighElt = N->getOperand(1); 6701 if (!isZeroNode(HighElt)) 6702 return SDValue(); 6703 6704 // Value must be a load. 6705 SDNode *Base = N->getOperand(0).Val; 6706 if (!isa<LoadSDNode>(Base)) { 6707 if (Base->getOpcode() != ISD::BIT_CONVERT) 6708 return SDValue(); 6709 Base = Base->getOperand(0).Val; 6710 if (!isa<LoadSDNode>(Base)) 6711 return SDValue(); 6712 } 6713 6714 // Transform it into VZEXT_LOAD addr. 6715 LoadSDNode *LD = cast<LoadSDNode>(Base); 6716 6717 // Load must not be an extload. 6718 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6719 return SDValue(); 6720 6721 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6722} 6723 6724/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6725static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6726 const X86Subtarget *Subtarget) { 6727 SDValue Cond = N->getOperand(0); 6728 6729 // If we have SSE[12] support, try to form min/max nodes. 6730 if (Subtarget->hasSSE2() && 6731 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6732 if (Cond.getOpcode() == ISD::SETCC) { 6733 // Get the LHS/RHS of the select. 6734 SDValue LHS = N->getOperand(1); 6735 SDValue RHS = N->getOperand(2); 6736 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6737 6738 unsigned Opcode = 0; 6739 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6740 switch (CC) { 6741 default: break; 6742 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6743 case ISD::SETULE: 6744 case ISD::SETLE: 6745 if (!UnsafeFPMath) break; 6746 // FALL THROUGH. 6747 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6748 case ISD::SETLT: 6749 Opcode = X86ISD::FMIN; 6750 break; 6751 6752 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6753 case ISD::SETUGT: 6754 case ISD::SETGT: 6755 if (!UnsafeFPMath) break; 6756 // FALL THROUGH. 6757 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6758 case ISD::SETGE: 6759 Opcode = X86ISD::FMAX; 6760 break; 6761 } 6762 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6763 switch (CC) { 6764 default: break; 6765 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6766 case ISD::SETUGT: 6767 case ISD::SETGT: 6768 if (!UnsafeFPMath) break; 6769 // FALL THROUGH. 6770 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6771 case ISD::SETGE: 6772 Opcode = X86ISD::FMIN; 6773 break; 6774 6775 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6776 case ISD::SETULE: 6777 case ISD::SETLE: 6778 if (!UnsafeFPMath) break; 6779 // FALL THROUGH. 6780 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6781 case ISD::SETLT: 6782 Opcode = X86ISD::FMAX; 6783 break; 6784 } 6785 } 6786 6787 if (Opcode) 6788 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6789 } 6790 6791 } 6792 6793 return SDValue(); 6794} 6795 6796/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6797static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6798 const X86Subtarget *Subtarget) { 6799 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6800 // the FP state in cases where an emms may be missing. 6801 // A preferable solution to the general problem is to figure out the right 6802 // places to insert EMMS. This qualifies as a quick hack. 6803 StoreSDNode *St = cast<StoreSDNode>(N); 6804 if (St->getValue().getValueType().isVector() && 6805 St->getValue().getValueType().getSizeInBits() == 64 && 6806 isa<LoadSDNode>(St->getValue()) && 6807 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6808 St->getChain().hasOneUse() && !St->isVolatile()) { 6809 SDNode* LdVal = St->getValue().Val; 6810 LoadSDNode *Ld = 0; 6811 int TokenFactorIndex = -1; 6812 SmallVector<SDValue, 8> Ops; 6813 SDNode* ChainVal = St->getChain().Val; 6814 // Must be a store of a load. We currently handle two cases: the load 6815 // is a direct child, and it's under an intervening TokenFactor. It is 6816 // possible to dig deeper under nested TokenFactors. 6817 if (ChainVal == LdVal) 6818 Ld = cast<LoadSDNode>(St->getChain()); 6819 else if (St->getValue().hasOneUse() && 6820 ChainVal->getOpcode() == ISD::TokenFactor) { 6821 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6822 if (ChainVal->getOperand(i).Val == LdVal) { 6823 TokenFactorIndex = i; 6824 Ld = cast<LoadSDNode>(St->getValue()); 6825 } else 6826 Ops.push_back(ChainVal->getOperand(i)); 6827 } 6828 } 6829 if (Ld) { 6830 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6831 if (Subtarget->is64Bit()) { 6832 SDValue NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6833 Ld->getBasePtr(), Ld->getSrcValue(), 6834 Ld->getSrcValueOffset(), Ld->isVolatile(), 6835 Ld->getAlignment()); 6836 SDValue NewChain = NewLd.getValue(1); 6837 if (TokenFactorIndex != -1) { 6838 Ops.push_back(NewChain); 6839 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6840 Ops.size()); 6841 } 6842 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6843 St->getSrcValue(), St->getSrcValueOffset(), 6844 St->isVolatile(), St->getAlignment()); 6845 } 6846 6847 // Otherwise, lower to two 32-bit copies. 6848 SDValue LoAddr = Ld->getBasePtr(); 6849 SDValue HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6850 DAG.getConstant(4, MVT::i32)); 6851 6852 SDValue LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6853 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6854 Ld->isVolatile(), Ld->getAlignment()); 6855 SDValue HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6856 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6857 Ld->isVolatile(), 6858 MinAlign(Ld->getAlignment(), 4)); 6859 6860 SDValue NewChain = LoLd.getValue(1); 6861 if (TokenFactorIndex != -1) { 6862 Ops.push_back(LoLd); 6863 Ops.push_back(HiLd); 6864 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6865 Ops.size()); 6866 } 6867 6868 LoAddr = St->getBasePtr(); 6869 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6870 DAG.getConstant(4, MVT::i32)); 6871 6872 SDValue LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6873 St->getSrcValue(), St->getSrcValueOffset(), 6874 St->isVolatile(), St->getAlignment()); 6875 SDValue HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6876 St->getSrcValue(), St->getSrcValueOffset()+4, 6877 St->isVolatile(), 6878 MinAlign(St->getAlignment(), 4)); 6879 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6880 } 6881 } 6882 return SDValue(); 6883} 6884 6885/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6886/// X86ISD::FXOR nodes. 6887static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6888 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6889 // F[X]OR(0.0, x) -> x 6890 // F[X]OR(x, 0.0) -> x 6891 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6892 if (C->getValueAPF().isPosZero()) 6893 return N->getOperand(1); 6894 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6895 if (C->getValueAPF().isPosZero()) 6896 return N->getOperand(0); 6897 return SDValue(); 6898} 6899 6900/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 6901static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 6902 // FAND(0.0, x) -> 0.0 6903 // FAND(x, 0.0) -> 0.0 6904 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6905 if (C->getValueAPF().isPosZero()) 6906 return N->getOperand(0); 6907 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6908 if (C->getValueAPF().isPosZero()) 6909 return N->getOperand(1); 6910 return SDValue(); 6911} 6912 6913 6914SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 6915 DAGCombinerInfo &DCI) const { 6916 SelectionDAG &DAG = DCI.DAG; 6917 switch (N->getOpcode()) { 6918 default: break; 6919 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 6920 case ISD::BUILD_VECTOR: 6921 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 6922 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 6923 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 6924 case X86ISD::FXOR: 6925 case X86ISD::FOR: return PerformFORCombine(N, DAG); 6926 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 6927 } 6928 6929 return SDValue(); 6930} 6931 6932//===----------------------------------------------------------------------===// 6933// X86 Inline Assembly Support 6934//===----------------------------------------------------------------------===// 6935 6936/// getConstraintType - Given a constraint letter, return the type of 6937/// constraint it is for this target. 6938X86TargetLowering::ConstraintType 6939X86TargetLowering::getConstraintType(const std::string &Constraint) const { 6940 if (Constraint.size() == 1) { 6941 switch (Constraint[0]) { 6942 case 'A': 6943 case 'f': 6944 case 'r': 6945 case 'R': 6946 case 'l': 6947 case 'q': 6948 case 'Q': 6949 case 'x': 6950 case 'y': 6951 case 'Y': 6952 return C_RegisterClass; 6953 default: 6954 break; 6955 } 6956 } 6957 return TargetLowering::getConstraintType(Constraint); 6958} 6959 6960/// LowerXConstraint - try to replace an X constraint, which matches anything, 6961/// with another that has more specific requirements based on the type of the 6962/// corresponding operand. 6963const char *X86TargetLowering:: 6964LowerXConstraint(MVT ConstraintVT) const { 6965 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 6966 // 'f' like normal targets. 6967 if (ConstraintVT.isFloatingPoint()) { 6968 if (Subtarget->hasSSE2()) 6969 return "Y"; 6970 if (Subtarget->hasSSE1()) 6971 return "x"; 6972 } 6973 6974 return TargetLowering::LowerXConstraint(ConstraintVT); 6975} 6976 6977/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 6978/// vector. If it is invalid, don't add anything to Ops. 6979void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 6980 char Constraint, 6981 std::vector<SDValue>&Ops, 6982 SelectionDAG &DAG) const { 6983 SDValue Result(0, 0); 6984 6985 switch (Constraint) { 6986 default: break; 6987 case 'I': 6988 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6989 if (C->getValue() <= 31) { 6990 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6991 break; 6992 } 6993 } 6994 return; 6995 case 'N': 6996 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 6997 if (C->getValue() <= 255) { 6998 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 6999 break; 7000 } 7001 } 7002 return; 7003 case 'i': { 7004 // Literal immediates are always ok. 7005 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 7006 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 7007 break; 7008 } 7009 7010 // If we are in non-pic codegen mode, we allow the address of a global (with 7011 // an optional displacement) to be used with 'i'. 7012 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 7013 int64_t Offset = 0; 7014 7015 // Match either (GA) or (GA+C) 7016 if (GA) { 7017 Offset = GA->getOffset(); 7018 } else if (Op.getOpcode() == ISD::ADD) { 7019 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7020 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7021 if (C && GA) { 7022 Offset = GA->getOffset()+C->getValue(); 7023 } else { 7024 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7025 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7026 if (C && GA) 7027 Offset = GA->getOffset()+C->getValue(); 7028 else 7029 C = 0, GA = 0; 7030 } 7031 } 7032 7033 if (GA) { 7034 // If addressing this global requires a load (e.g. in PIC mode), we can't 7035 // match. 7036 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 7037 false)) 7038 return; 7039 7040 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 7041 Offset); 7042 Result = Op; 7043 break; 7044 } 7045 7046 // Otherwise, not valid for this mode. 7047 return; 7048 } 7049 } 7050 7051 if (Result.Val) { 7052 Ops.push_back(Result); 7053 return; 7054 } 7055 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 7056} 7057 7058std::vector<unsigned> X86TargetLowering:: 7059getRegClassForInlineAsmConstraint(const std::string &Constraint, 7060 MVT VT) const { 7061 if (Constraint.size() == 1) { 7062 // FIXME: not handling fp-stack yet! 7063 switch (Constraint[0]) { // GCC X86 Constraint Letters 7064 default: break; // Unknown constraint letter 7065 case 'A': // EAX/EDX 7066 if (VT == MVT::i32 || VT == MVT::i64) 7067 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 7068 break; 7069 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 7070 case 'Q': // Q_REGS 7071 if (VT == MVT::i32) 7072 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 7073 else if (VT == MVT::i16) 7074 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 7075 else if (VT == MVT::i8) 7076 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 7077 else if (VT == MVT::i64) 7078 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 7079 break; 7080 } 7081 } 7082 7083 return std::vector<unsigned>(); 7084} 7085 7086std::pair<unsigned, const TargetRegisterClass*> 7087X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 7088 MVT VT) const { 7089 // First, see if this is a constraint that directly corresponds to an LLVM 7090 // register class. 7091 if (Constraint.size() == 1) { 7092 // GCC Constraint Letters 7093 switch (Constraint[0]) { 7094 default: break; 7095 case 'r': // GENERAL_REGS 7096 case 'R': // LEGACY_REGS 7097 case 'l': // INDEX_REGS 7098 if (VT == MVT::i64 && Subtarget->is64Bit()) 7099 return std::make_pair(0U, X86::GR64RegisterClass); 7100 if (VT == MVT::i32) 7101 return std::make_pair(0U, X86::GR32RegisterClass); 7102 else if (VT == MVT::i16) 7103 return std::make_pair(0U, X86::GR16RegisterClass); 7104 else if (VT == MVT::i8) 7105 return std::make_pair(0U, X86::GR8RegisterClass); 7106 break; 7107 case 'f': // FP Stack registers. 7108 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 7109 // value to the correct fpstack register class. 7110 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 7111 return std::make_pair(0U, X86::RFP32RegisterClass); 7112 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 7113 return std::make_pair(0U, X86::RFP64RegisterClass); 7114 return std::make_pair(0U, X86::RFP80RegisterClass); 7115 case 'y': // MMX_REGS if MMX allowed. 7116 if (!Subtarget->hasMMX()) break; 7117 return std::make_pair(0U, X86::VR64RegisterClass); 7118 break; 7119 case 'Y': // SSE_REGS if SSE2 allowed 7120 if (!Subtarget->hasSSE2()) break; 7121 // FALL THROUGH. 7122 case 'x': // SSE_REGS if SSE1 allowed 7123 if (!Subtarget->hasSSE1()) break; 7124 7125 switch (VT.getSimpleVT()) { 7126 default: break; 7127 // Scalar SSE types. 7128 case MVT::f32: 7129 case MVT::i32: 7130 return std::make_pair(0U, X86::FR32RegisterClass); 7131 case MVT::f64: 7132 case MVT::i64: 7133 return std::make_pair(0U, X86::FR64RegisterClass); 7134 // Vector types. 7135 case MVT::v16i8: 7136 case MVT::v8i16: 7137 case MVT::v4i32: 7138 case MVT::v2i64: 7139 case MVT::v4f32: 7140 case MVT::v2f64: 7141 return std::make_pair(0U, X86::VR128RegisterClass); 7142 } 7143 break; 7144 } 7145 } 7146 7147 // Use the default implementation in TargetLowering to convert the register 7148 // constraint into a member of a register class. 7149 std::pair<unsigned, const TargetRegisterClass*> Res; 7150 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 7151 7152 // Not found as a standard register? 7153 if (Res.second == 0) { 7154 // GCC calls "st(0)" just plain "st". 7155 if (StringsEqualNoCase("{st}", Constraint)) { 7156 Res.first = X86::ST0; 7157 Res.second = X86::RFP80RegisterClass; 7158 } 7159 7160 return Res; 7161 } 7162 7163 // Otherwise, check to see if this is a register class of the wrong value 7164 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 7165 // turn into {ax},{dx}. 7166 if (Res.second->hasType(VT)) 7167 return Res; // Correct type already, nothing to do. 7168 7169 // All of the single-register GCC register classes map their values onto 7170 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 7171 // really want an 8-bit or 32-bit register, map to the appropriate register 7172 // class and return the appropriate register. 7173 if (Res.second != X86::GR16RegisterClass) 7174 return Res; 7175 7176 if (VT == MVT::i8) { 7177 unsigned DestReg = 0; 7178 switch (Res.first) { 7179 default: break; 7180 case X86::AX: DestReg = X86::AL; break; 7181 case X86::DX: DestReg = X86::DL; break; 7182 case X86::CX: DestReg = X86::CL; break; 7183 case X86::BX: DestReg = X86::BL; break; 7184 } 7185 if (DestReg) { 7186 Res.first = DestReg; 7187 Res.second = Res.second = X86::GR8RegisterClass; 7188 } 7189 } else if (VT == MVT::i32) { 7190 unsigned DestReg = 0; 7191 switch (Res.first) { 7192 default: break; 7193 case X86::AX: DestReg = X86::EAX; break; 7194 case X86::DX: DestReg = X86::EDX; break; 7195 case X86::CX: DestReg = X86::ECX; break; 7196 case X86::BX: DestReg = X86::EBX; break; 7197 case X86::SI: DestReg = X86::ESI; break; 7198 case X86::DI: DestReg = X86::EDI; break; 7199 case X86::BP: DestReg = X86::EBP; break; 7200 case X86::SP: DestReg = X86::ESP; break; 7201 } 7202 if (DestReg) { 7203 Res.first = DestReg; 7204 Res.second = Res.second = X86::GR32RegisterClass; 7205 } 7206 } else if (VT == MVT::i64) { 7207 unsigned DestReg = 0; 7208 switch (Res.first) { 7209 default: break; 7210 case X86::AX: DestReg = X86::RAX; break; 7211 case X86::DX: DestReg = X86::RDX; break; 7212 case X86::CX: DestReg = X86::RCX; break; 7213 case X86::BX: DestReg = X86::RBX; break; 7214 case X86::SI: DestReg = X86::RSI; break; 7215 case X86::DI: DestReg = X86::RDI; break; 7216 case X86::BP: DestReg = X86::RBP; break; 7217 case X86::SP: DestReg = X86::RSP; break; 7218 } 7219 if (DestReg) { 7220 Res.first = DestReg; 7221 Res.second = Res.second = X86::GR64RegisterClass; 7222 } 7223 } 7224 7225 return Res; 7226} 7227