X86ISelLowering.cpp revision a425e0073df51ab99665062a8c00d704e89d2ef8
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86TargetMachine.h" 20#include "llvm/CallingConv.h" 21#include "llvm/Constants.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/GlobalVariable.h" 24#include "llvm/Function.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/ADT/BitVector.h" 27#include "llvm/ADT/VectorExtras.h" 28#include "llvm/CodeGen/CallingConvLower.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/PseudoSourceValue.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/Debug.h" 38#include "llvm/Target/TargetOptions.h" 39#include "llvm/ADT/SmallSet.h" 40#include "llvm/ADT/StringExtras.h" 41using namespace llvm; 42 43// Forward declarations. 44static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG); 45 46X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 47 : TargetLowering(TM) { 48 Subtarget = &TM.getSubtarget<X86Subtarget>(); 49 X86ScalarSSEf64 = Subtarget->hasSSE2(); 50 X86ScalarSSEf32 = Subtarget->hasSSE1(); 51 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 52 53 bool Fast = false; 54 55 RegInfo = TM.getRegisterInfo(); 56 57 // Set up the TargetLowering object. 58 59 // X86 is weird, it always uses i8 for shift amounts and setcc results. 60 setShiftAmountType(MVT::i8); 61 setSetCCResultContents(ZeroOrOneSetCCResult); 62 setSchedulingPreference(SchedulingForRegPressure); 63 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 64 setStackPointerRegisterToSaveRestore(X86StackPtr); 65 66 if (Subtarget->isTargetDarwin()) { 67 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 68 setUseUnderscoreSetJmp(false); 69 setUseUnderscoreLongJmp(false); 70 } else if (Subtarget->isTargetMingw()) { 71 // MS runtime is weird: it exports _setjmp, but longjmp! 72 setUseUnderscoreSetJmp(true); 73 setUseUnderscoreLongJmp(false); 74 } else { 75 setUseUnderscoreSetJmp(true); 76 setUseUnderscoreLongJmp(true); 77 } 78 79 // Set up the register classes. 80 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 81 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 82 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 83 if (Subtarget->is64Bit()) 84 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 85 86 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote); 87 88 // We don't accept any truncstore of integer registers. 89 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 90 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 91 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 92 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 93 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 94 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 95 96 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 97 // operation. 98 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 99 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 100 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 101 102 if (Subtarget->is64Bit()) { 103 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 104 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 105 } else { 106 if (X86ScalarSSEf64) 107 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. 108 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 109 else 110 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 111 } 112 113 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 114 // this operation. 115 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 116 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 117 // SSE has no i16 to fp conversion, only i32 118 if (X86ScalarSSEf32) { 119 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 120 // f32 and f64 cases are Legal, f80 case is not 121 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 122 } else { 123 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 124 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 125 } 126 127 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 128 // are Legal, f80 is custom lowered. 129 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 130 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 131 132 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 133 // this operation. 134 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 135 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 136 137 if (X86ScalarSSEf32) { 138 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 139 // f32 and f64 cases are Legal, f80 case is not 140 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 141 } else { 142 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 143 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 144 } 145 146 // Handle FP_TO_UINT by promoting the destination to a larger signed 147 // conversion. 148 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 149 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 150 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 151 152 if (Subtarget->is64Bit()) { 153 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 155 } else { 156 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 157 // Expand FP_TO_UINT into a select. 158 // FIXME: We would like to use a Custom expander here eventually to do 159 // the optimal thing for SSE vs. the default expansion in the legalizer. 160 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 161 else 162 // With SSE3 we can use fisttpll to convert to a signed i64. 163 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 164 } 165 166 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 167 if (!X86ScalarSSEf64) { 168 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 169 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 170 } 171 172 // Scalar integer divide and remainder are lowered to use operations that 173 // produce two results, to match the available instructions. This exposes 174 // the two-result form to trivial CSE, which is able to combine x/y and x%y 175 // into a single instruction. 176 // 177 // Scalar integer multiply-high is also lowered to use two-result 178 // operations, to match the available instructions. However, plain multiply 179 // (low) operations are left as Legal, as there are single-result 180 // instructions for this in x86. Using the two-result multiply instructions 181 // when both high and low results are needed must be arranged by dagcombine. 182 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 183 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 184 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 185 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 186 setOperationAction(ISD::SREM , MVT::i8 , Expand); 187 setOperationAction(ISD::UREM , MVT::i8 , Expand); 188 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 189 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 190 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 191 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 192 setOperationAction(ISD::SREM , MVT::i16 , Expand); 193 setOperationAction(ISD::UREM , MVT::i16 , Expand); 194 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 195 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 196 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 197 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 198 setOperationAction(ISD::SREM , MVT::i32 , Expand); 199 setOperationAction(ISD::UREM , MVT::i32 , Expand); 200 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 201 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 202 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 203 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 204 setOperationAction(ISD::SREM , MVT::i64 , Expand); 205 setOperationAction(ISD::UREM , MVT::i64 , Expand); 206 207 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 208 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 209 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 210 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 211 if (Subtarget->is64Bit()) 212 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 213 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 215 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 216 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 217 setOperationAction(ISD::FREM , MVT::f32 , Expand); 218 setOperationAction(ISD::FREM , MVT::f64 , Expand); 219 setOperationAction(ISD::FREM , MVT::f80 , Expand); 220 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 221 222 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 223 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 224 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 225 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 226 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 227 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 228 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 229 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 230 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 231 if (Subtarget->is64Bit()) { 232 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 233 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 234 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 235 } 236 237 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 238 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 239 240 // These should be promoted to a larger select which is supported. 241 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 242 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 243 // X86 wants to expand cmov itself. 244 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 245 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 246 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 247 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 248 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 249 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 250 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 251 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 252 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 253 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 254 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 255 if (Subtarget->is64Bit()) { 256 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 257 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 258 } 259 // X86 ret instruction may pop stack. 260 setOperationAction(ISD::RET , MVT::Other, Custom); 261 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 262 263 // Darwin ABI issue. 264 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 265 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 266 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 267 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 268 if (Subtarget->is64Bit()) 269 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 270 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 271 if (Subtarget->is64Bit()) { 272 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 273 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 274 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 275 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 276 } 277 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 278 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 279 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 280 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 281 if (Subtarget->is64Bit()) { 282 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 283 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 284 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 285 } 286 287 if (Subtarget->hasSSE1()) 288 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 289 290 if (!Subtarget->hasSSE2()) 291 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 292 293 // Expand certain atomics 294 setOperationAction(ISD::ATOMIC_CMP_SWAP_8 , MVT::i8, Custom); 295 setOperationAction(ISD::ATOMIC_CMP_SWAP_16, MVT::i16, Custom); 296 setOperationAction(ISD::ATOMIC_CMP_SWAP_32, MVT::i32, Custom); 297 setOperationAction(ISD::ATOMIC_CMP_SWAP_64, MVT::i64, Custom); 298 299 setOperationAction(ISD::ATOMIC_LOAD_SUB_8, MVT::i8, Expand); 300 setOperationAction(ISD::ATOMIC_LOAD_SUB_16, MVT::i16, Expand); 301 setOperationAction(ISD::ATOMIC_LOAD_SUB_32, MVT::i32, Expand); 302 setOperationAction(ISD::ATOMIC_LOAD_SUB_64, MVT::i64, Expand); 303 304 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 305 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 306 // FIXME - use subtarget debug flags 307 if (!Subtarget->isTargetDarwin() && 308 !Subtarget->isTargetELF() && 309 !Subtarget->isTargetCygMing()) { 310 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 311 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 312 } 313 314 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 315 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 316 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 317 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 318 if (Subtarget->is64Bit()) { 319 // FIXME: Verify 320 setExceptionPointerRegister(X86::RAX); 321 setExceptionSelectorRegister(X86::RDX); 322 } else { 323 setExceptionPointerRegister(X86::EAX); 324 setExceptionSelectorRegister(X86::EDX); 325 } 326 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 327 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 328 329 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 330 331 setOperationAction(ISD::TRAP, MVT::Other, Legal); 332 333 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 334 setOperationAction(ISD::VASTART , MVT::Other, Custom); 335 setOperationAction(ISD::VAEND , MVT::Other, Expand); 336 if (Subtarget->is64Bit()) { 337 setOperationAction(ISD::VAARG , MVT::Other, Custom); 338 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 339 } else { 340 setOperationAction(ISD::VAARG , MVT::Other, Expand); 341 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 342 } 343 344 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 345 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 346 if (Subtarget->is64Bit()) 347 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 348 if (Subtarget->isTargetCygMing()) 349 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 350 else 351 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 352 353 if (X86ScalarSSEf64) { 354 // f32 and f64 use SSE. 355 // Set up the FP register classes. 356 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 357 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 358 359 // Use ANDPD to simulate FABS. 360 setOperationAction(ISD::FABS , MVT::f64, Custom); 361 setOperationAction(ISD::FABS , MVT::f32, Custom); 362 363 // Use XORP to simulate FNEG. 364 setOperationAction(ISD::FNEG , MVT::f64, Custom); 365 setOperationAction(ISD::FNEG , MVT::f32, Custom); 366 367 // Use ANDPD and ORPD to simulate FCOPYSIGN. 368 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 369 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 370 371 // We don't support sin/cos/fmod 372 setOperationAction(ISD::FSIN , MVT::f64, Expand); 373 setOperationAction(ISD::FCOS , MVT::f64, Expand); 374 setOperationAction(ISD::FSIN , MVT::f32, Expand); 375 setOperationAction(ISD::FCOS , MVT::f32, Expand); 376 377 // Expand FP immediates into loads from the stack, except for the special 378 // cases we handle. 379 addLegalFPImmediate(APFloat(+0.0)); // xorpd 380 addLegalFPImmediate(APFloat(+0.0f)); // xorps 381 382 // Floating truncations from f80 and extensions to f80 go through memory. 383 // If optimizing, we lie about this though and handle it in 384 // InstructionSelectPreprocess so that dagcombine2 can hack on these. 385 if (Fast) { 386 setConvertAction(MVT::f32, MVT::f80, Expand); 387 setConvertAction(MVT::f64, MVT::f80, Expand); 388 setConvertAction(MVT::f80, MVT::f32, Expand); 389 setConvertAction(MVT::f80, MVT::f64, Expand); 390 } 391 } else if (X86ScalarSSEf32) { 392 // Use SSE for f32, x87 for f64. 393 // Set up the FP register classes. 394 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 395 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 396 397 // Use ANDPS to simulate FABS. 398 setOperationAction(ISD::FABS , MVT::f32, Custom); 399 400 // Use XORP to simulate FNEG. 401 setOperationAction(ISD::FNEG , MVT::f32, Custom); 402 403 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 404 405 // Use ANDPS and ORPS to simulate FCOPYSIGN. 406 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 407 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 408 409 // We don't support sin/cos/fmod 410 setOperationAction(ISD::FSIN , MVT::f32, Expand); 411 setOperationAction(ISD::FCOS , MVT::f32, Expand); 412 413 // Special cases we handle for FP constants. 414 addLegalFPImmediate(APFloat(+0.0f)); // xorps 415 addLegalFPImmediate(APFloat(+0.0)); // FLD0 416 addLegalFPImmediate(APFloat(+1.0)); // FLD1 417 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 418 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 419 420 // SSE <-> X87 conversions go through memory. If optimizing, we lie about 421 // this though and handle it in InstructionSelectPreprocess so that 422 // dagcombine2 can hack on these. 423 if (Fast) { 424 setConvertAction(MVT::f32, MVT::f64, Expand); 425 setConvertAction(MVT::f32, MVT::f80, Expand); 426 setConvertAction(MVT::f80, MVT::f32, Expand); 427 setConvertAction(MVT::f64, MVT::f32, Expand); 428 // And x87->x87 truncations also. 429 setConvertAction(MVT::f80, MVT::f64, Expand); 430 } 431 432 if (!UnsafeFPMath) { 433 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 434 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 435 } 436 } else { 437 // f32 and f64 in x87. 438 // Set up the FP register classes. 439 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 440 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 441 442 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 443 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 444 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 445 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 446 447 // Floating truncations go through memory. If optimizing, we lie about 448 // this though and handle it in InstructionSelectPreprocess so that 449 // dagcombine2 can hack on these. 450 if (Fast) { 451 setConvertAction(MVT::f80, MVT::f32, Expand); 452 setConvertAction(MVT::f64, MVT::f32, Expand); 453 setConvertAction(MVT::f80, MVT::f64, Expand); 454 } 455 456 if (!UnsafeFPMath) { 457 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 458 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 459 } 460 addLegalFPImmediate(APFloat(+0.0)); // FLD0 461 addLegalFPImmediate(APFloat(+1.0)); // FLD1 462 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 463 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 464 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 465 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 466 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 467 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 468 } 469 470 // Long double always uses X87. 471 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 472 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 473 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 474 { 475 APFloat TmpFlt(+0.0); 476 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 477 addLegalFPImmediate(TmpFlt); // FLD0 478 TmpFlt.changeSign(); 479 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 480 APFloat TmpFlt2(+1.0); 481 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven); 482 addLegalFPImmediate(TmpFlt2); // FLD1 483 TmpFlt2.changeSign(); 484 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 485 } 486 487 if (!UnsafeFPMath) { 488 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 489 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 490 } 491 492 // Always use a library call for pow. 493 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 494 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 495 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 496 497 setOperationAction(ISD::FLOG, MVT::f32, Expand); 498 setOperationAction(ISD::FLOG, MVT::f64, Expand); 499 setOperationAction(ISD::FLOG, MVT::f80, Expand); 500 setOperationAction(ISD::FLOG2, MVT::f32, Expand); 501 setOperationAction(ISD::FLOG2, MVT::f64, Expand); 502 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 503 setOperationAction(ISD::FLOG10, MVT::f32, Expand); 504 setOperationAction(ISD::FLOG10, MVT::f64, Expand); 505 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 506 setOperationAction(ISD::FEXP, MVT::f32, Expand); 507 setOperationAction(ISD::FEXP, MVT::f64, Expand); 508 setOperationAction(ISD::FEXP, MVT::f80, Expand); 509 setOperationAction(ISD::FEXP2, MVT::f32, Expand); 510 setOperationAction(ISD::FEXP2, MVT::f64, Expand); 511 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 512 513 // First set operation action for all vector types to expand. Then we 514 // will selectively turn on ones that can be effectively codegen'd. 515 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 516 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 517 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 518 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 519 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 520 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 521 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 522 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 523 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 524 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 525 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 526 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 527 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 528 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 529 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 530 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 531 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); 532 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); 533 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 534 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 535 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 536 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 537 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 538 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 539 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 540 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 541 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 542 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 543 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 544 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 545 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 546 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 547 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 548 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 549 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 550 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 551 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 552 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 553 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 554 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 555 } 556 557 if (Subtarget->hasMMX()) { 558 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 559 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 560 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 561 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 562 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 563 564 // FIXME: add MMX packed arithmetics 565 566 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 567 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 568 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 569 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 570 571 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 572 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 573 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 574 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 575 576 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 577 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 578 579 setOperationAction(ISD::AND, MVT::v8i8, Promote); 580 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 581 setOperationAction(ISD::AND, MVT::v4i16, Promote); 582 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 583 setOperationAction(ISD::AND, MVT::v2i32, Promote); 584 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 585 setOperationAction(ISD::AND, MVT::v1i64, Legal); 586 587 setOperationAction(ISD::OR, MVT::v8i8, Promote); 588 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 589 setOperationAction(ISD::OR, MVT::v4i16, Promote); 590 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 591 setOperationAction(ISD::OR, MVT::v2i32, Promote); 592 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 593 setOperationAction(ISD::OR, MVT::v1i64, Legal); 594 595 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 596 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 597 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 598 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 599 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 600 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 601 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 602 603 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 604 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 605 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 606 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 607 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 608 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 609 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 610 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 611 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 612 613 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 614 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 615 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 616 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 617 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 618 619 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 620 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 621 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 622 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 623 624 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); 625 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 626 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 627 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 628 629 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 630 } 631 632 if (Subtarget->hasSSE1()) { 633 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 634 635 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 636 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 637 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 638 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 639 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 640 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 641 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 642 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 643 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 644 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 645 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 646 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 647 } 648 649 if (Subtarget->hasSSE2()) { 650 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 651 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 652 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 653 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 654 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 655 656 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 657 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 658 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 659 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 660 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 661 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 662 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 663 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 664 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 665 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 666 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 667 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 668 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 669 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 670 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 671 672 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 673 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 674 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 675 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 676 677 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 678 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 679 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 680 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 681 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 682 683 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 684 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 685 MVT VT = (MVT::SimpleValueType)i; 686 // Do not attempt to custom lower non-power-of-2 vectors 687 if (!isPowerOf2_32(VT.getVectorNumElements())) 688 continue; 689 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 690 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 691 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 692 } 693 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 694 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 695 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 696 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 697 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 698 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 699 if (Subtarget->is64Bit()) { 700 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 701 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 702 } 703 704 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 705 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { 706 setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); 707 AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); 708 setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); 709 AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); 710 setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); 711 AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); 712 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); 713 AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); 714 setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); 715 AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); 716 } 717 718 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 719 720 // Custom lower v2i64 and v2f64 selects. 721 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 722 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 723 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 724 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 725 726 } 727 728 if (Subtarget->hasSSE41()) { 729 // FIXME: Do we need to handle scalar-to-vector here? 730 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 731 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 732 733 // i8 and i16 vectors are custom , because the source register and source 734 // source memory operand types are not the same width. f32 vectors are 735 // custom since the immediate controlling the insert encodes additional 736 // information. 737 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 738 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 739 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 740 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 741 742 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 743 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 744 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 745 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 746 747 if (Subtarget->is64Bit()) { 748 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 749 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 750 } 751 } 752 753 if (Subtarget->hasSSE42()) { 754 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 755 } 756 757 // We want to custom lower some of our intrinsics. 758 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 759 760 // We have target-specific dag combine patterns for the following nodes: 761 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 762 setTargetDAGCombine(ISD::BUILD_VECTOR); 763 setTargetDAGCombine(ISD::SELECT); 764 setTargetDAGCombine(ISD::STORE); 765 766 computeRegisterProperties(); 767 768 // FIXME: These should be based on subtarget info. Plus, the values should 769 // be smaller when we are in optimizing for size mode. 770 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 771 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 772 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 773 allowUnalignedMemoryAccesses = true; // x86 supports it! 774 setPrefLoopAlignment(16); 775} 776 777 778MVT X86TargetLowering::getSetCCResultType(const SDValue &) const { 779 return MVT::i8; 780} 781 782 783/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 784/// the desired ByVal argument alignment. 785static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 786 if (MaxAlign == 16) 787 return; 788 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 789 if (VTy->getBitWidth() == 128) 790 MaxAlign = 16; 791 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 792 unsigned EltAlign = 0; 793 getMaxByValAlign(ATy->getElementType(), EltAlign); 794 if (EltAlign > MaxAlign) 795 MaxAlign = EltAlign; 796 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 797 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 798 unsigned EltAlign = 0; 799 getMaxByValAlign(STy->getElementType(i), EltAlign); 800 if (EltAlign > MaxAlign) 801 MaxAlign = EltAlign; 802 if (MaxAlign == 16) 803 break; 804 } 805 } 806 return; 807} 808 809/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 810/// function arguments in the caller parameter area. For X86, aggregates 811/// that contain SSE vectors are placed at 16-byte boundaries while the rest 812/// are at 4-byte boundaries. 813unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 814 if (Subtarget->is64Bit()) { 815 // Max of 8 and alignment of type. 816 unsigned TyAlign = getTargetData()->getABITypeAlignment(Ty); 817 if (TyAlign > 8) 818 return TyAlign; 819 return 8; 820 } 821 822 unsigned Align = 4; 823 if (Subtarget->hasSSE1()) 824 getMaxByValAlign(Ty, Align); 825 return Align; 826} 827 828/// getOptimalMemOpType - Returns the target specific optimal type for load 829/// and store operations as a result of memset, memcpy, and memmove 830/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 831/// determining it. 832MVT 833X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 834 bool isSrcConst, bool isSrcStr) const { 835 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 836 return MVT::v4i32; 837 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 838 return MVT::v4f32; 839 if (Subtarget->is64Bit() && Size >= 8) 840 return MVT::i64; 841 return MVT::i32; 842} 843 844 845/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 846/// jumptable. 847SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 848 SelectionDAG &DAG) const { 849 if (usesGlobalOffsetTable()) 850 return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); 851 if (!Subtarget->isPICStyleRIPRel()) 852 return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); 853 return Table; 854} 855 856//===----------------------------------------------------------------------===// 857// Return Value Calling Convention Implementation 858//===----------------------------------------------------------------------===// 859 860#include "X86GenCallingConv.inc" 861 862/// LowerRET - Lower an ISD::RET node. 863SDValue X86TargetLowering::LowerRET(SDValue Op, SelectionDAG &DAG) { 864 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); 865 866 SmallVector<CCValAssign, 16> RVLocs; 867 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); 868 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 869 CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); 870 CCInfo.AnalyzeReturn(Op.getNode(), RetCC_X86); 871 872 // If this is the first return lowered for this function, add the regs to the 873 // liveout set for the function. 874 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 875 for (unsigned i = 0; i != RVLocs.size(); ++i) 876 if (RVLocs[i].isRegLoc()) 877 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 878 } 879 SDValue Chain = Op.getOperand(0); 880 881 // Handle tail call return. 882 Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); 883 if (Chain.getOpcode() == X86ISD::TAILCALL) { 884 SDValue TailCall = Chain; 885 SDValue TargetAddress = TailCall.getOperand(1); 886 SDValue StackAdjustment = TailCall.getOperand(2); 887 assert(((TargetAddress.getOpcode() == ISD::Register && 888 (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::ECX || 889 cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || 890 TargetAddress.getOpcode() == ISD::TargetExternalSymbol || 891 TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && 892 "Expecting an global address, external symbol, or register"); 893 assert(StackAdjustment.getOpcode() == ISD::Constant && 894 "Expecting a const value"); 895 896 SmallVector<SDValue,8> Operands; 897 Operands.push_back(Chain.getOperand(0)); 898 Operands.push_back(TargetAddress); 899 Operands.push_back(StackAdjustment); 900 // Copy registers used by the call. Last operand is a flag so it is not 901 // copied. 902 for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { 903 Operands.push_back(Chain.getOperand(i)); 904 } 905 return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], 906 Operands.size()); 907 } 908 909 // Regular return. 910 SDValue Flag; 911 912 SmallVector<SDValue, 6> RetOps; 913 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 914 // Operand #1 = Bytes To Pop 915 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 916 917 // Copy the result values into the output registers. 918 for (unsigned i = 0; i != RVLocs.size(); ++i) { 919 CCValAssign &VA = RVLocs[i]; 920 assert(VA.isRegLoc() && "Can only return in registers!"); 921 SDValue ValToCopy = Op.getOperand(i*2+1); 922 923 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 924 // the RET instruction and handled by the FP Stackifier. 925 if (RVLocs[i].getLocReg() == X86::ST0 || 926 RVLocs[i].getLocReg() == X86::ST1) { 927 // If this is a copy from an xmm register to ST(0), use an FPExtend to 928 // change the value to the FP stack register class. 929 if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) 930 ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); 931 RetOps.push_back(ValToCopy); 932 // Don't emit a copytoreg. 933 continue; 934 } 935 936 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); 937 Flag = Chain.getValue(1); 938 } 939 940 // The x86-64 ABI for returning structs by value requires that we copy 941 // the sret argument into %rax for the return. We saved the argument into 942 // a virtual register in the entry block, so now we copy the value out 943 // and into %rax. 944 if (Subtarget->is64Bit() && 945 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 946 MachineFunction &MF = DAG.getMachineFunction(); 947 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 948 unsigned Reg = FuncInfo->getSRetReturnReg(); 949 if (!Reg) { 950 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 951 FuncInfo->setSRetReturnReg(Reg); 952 } 953 SDValue Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); 954 955 Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); 956 Flag = Chain.getValue(1); 957 } 958 959 RetOps[0] = Chain; // Update chain. 960 961 // Add the flag if we have it. 962 if (Flag.getNode()) 963 RetOps.push_back(Flag); 964 965 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); 966} 967 968 969/// LowerCallResult - Lower the result values of an ISD::CALL into the 970/// appropriate copies out of appropriate physical registers. This assumes that 971/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call 972/// being lowered. The returns a SDNode with the same number of values as the 973/// ISD::CALL. 974SDNode *X86TargetLowering:: 975LowerCallResult(SDValue Chain, SDValue InFlag, SDNode *TheCall, 976 unsigned CallingConv, SelectionDAG &DAG) { 977 978 // Assign locations to each value returned by this call. 979 SmallVector<CCValAssign, 16> RVLocs; 980 bool isVarArg = cast<ConstantSDNode>(TheCall->getOperand(2))->getValue() != 0; 981 CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); 982 CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); 983 984 SmallVector<SDValue, 8> ResultVals; 985 986 // Copy all of the result registers out of their specified physreg. 987 for (unsigned i = 0; i != RVLocs.size(); ++i) { 988 MVT CopyVT = RVLocs[i].getValVT(); 989 990 // If this is a call to a function that returns an fp value on the floating 991 // point stack, but where we prefer to use the value in xmm registers, copy 992 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 993 if ((RVLocs[i].getLocReg() == X86::ST0 || 994 RVLocs[i].getLocReg() == X86::ST1) && 995 isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { 996 CopyVT = MVT::f80; 997 } 998 999 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), 1000 CopyVT, InFlag).getValue(1); 1001 SDValue Val = Chain.getValue(0); 1002 InFlag = Chain.getValue(2); 1003 1004 if (CopyVT != RVLocs[i].getValVT()) { 1005 // Round the F80 the right size, which also moves to the appropriate xmm 1006 // register. 1007 Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, 1008 // This truncation won't change the value. 1009 DAG.getIntPtrConstant(1)); 1010 } 1011 1012 ResultVals.push_back(Val); 1013 } 1014 1015 // Merge everything together with a MERGE_VALUES node. 1016 ResultVals.push_back(Chain); 1017 return DAG.getMergeValues(TheCall->getVTList(), &ResultVals[0], 1018 ResultVals.size()).getNode(); 1019} 1020 1021 1022//===----------------------------------------------------------------------===// 1023// C & StdCall & Fast Calling Convention implementation 1024//===----------------------------------------------------------------------===// 1025// StdCall calling convention seems to be standard for many Windows' API 1026// routines and around. It differs from C calling convention just a little: 1027// callee should clean up the stack, not caller. Symbols should be also 1028// decorated in some fancy way :) It doesn't support any vector arguments. 1029// For info on fast calling convention see Fast Calling Convention (tail call) 1030// implementation LowerX86_32FastCCCallTo. 1031 1032/// AddLiveIn - This helper function adds the specified physical register to the 1033/// MachineFunction as a live in value. It also creates a corresponding virtual 1034/// register for it. 1035static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 1036 const TargetRegisterClass *RC) { 1037 assert(RC->contains(PReg) && "Not the correct regclass!"); 1038 unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); 1039 MF.getRegInfo().addLiveIn(PReg, VReg); 1040 return VReg; 1041} 1042 1043/// CallIsStructReturn - Determines whether a CALL node uses struct return 1044/// semantics. 1045static bool CallIsStructReturn(SDValue Op) { 1046 unsigned NumOps = (Op.getNumOperands() - 5) / 2; 1047 if (!NumOps) 1048 return false; 1049 1050 return cast<ARG_FLAGSSDNode>(Op.getOperand(6))->getArgFlags().isSRet(); 1051} 1052 1053/// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct 1054/// return semantics. 1055static bool ArgsAreStructReturn(SDValue Op) { 1056 unsigned NumArgs = Op.getNode()->getNumValues() - 1; 1057 if (!NumArgs) 1058 return false; 1059 1060 return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); 1061} 1062 1063/// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires 1064/// the callee to pop its own arguments. Callee pop is necessary to support tail 1065/// calls. 1066bool X86TargetLowering::IsCalleePop(SDValue Op) { 1067 bool IsVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1068 if (IsVarArg) 1069 return false; 1070 1071 switch (cast<ConstantSDNode>(Op.getOperand(1))->getValue()) { 1072 default: 1073 return false; 1074 case CallingConv::X86_StdCall: 1075 return !Subtarget->is64Bit(); 1076 case CallingConv::X86_FastCall: 1077 return !Subtarget->is64Bit(); 1078 case CallingConv::Fast: 1079 return PerformTailCallOpt; 1080 } 1081} 1082 1083/// CCAssignFnForNode - Selects the correct CCAssignFn for a CALL or 1084/// FORMAL_ARGUMENTS node. 1085CCAssignFn *X86TargetLowering::CCAssignFnForNode(SDValue Op) const { 1086 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1087 1088 if (Subtarget->is64Bit()) { 1089 if (Subtarget->isTargetWin64()) 1090 return CC_X86_Win64_C; 1091 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1092 return CC_X86_64_TailCall; 1093 else 1094 return CC_X86_64_C; 1095 } 1096 1097 if (CC == CallingConv::X86_FastCall) 1098 return CC_X86_32_FastCall; 1099 else if (CC == CallingConv::Fast && PerformTailCallOpt) 1100 return CC_X86_32_TailCall; 1101 else if (CC == CallingConv::Fast) 1102 return CC_X86_32_FastCC; 1103 else 1104 return CC_X86_32_C; 1105} 1106 1107/// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to 1108/// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. 1109NameDecorationStyle 1110X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDValue Op) { 1111 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1112 if (CC == CallingConv::X86_FastCall) 1113 return FastCall; 1114 else if (CC == CallingConv::X86_StdCall) 1115 return StdCall; 1116 return None; 1117} 1118 1119 1120/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer 1121/// in a register before calling. 1122bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { 1123 return !IsTailCall && !Is64Bit && 1124 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1125 Subtarget->isPICStyleGOT(); 1126} 1127 1128/// CallRequiresFnAddressInReg - Check whether the call requires the function 1129/// address to be loaded in a register. 1130bool 1131X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { 1132 return !Is64Bit && IsTailCall && 1133 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1134 Subtarget->isPICStyleGOT(); 1135} 1136 1137/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 1138/// by "Src" to address "Dst" with size and alignment information specified by 1139/// the specific parameter attribute. The copy will be passed as a byval 1140/// function parameter. 1141static SDValue 1142CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 1143 ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { 1144 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 1145 return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), 1146 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 1147} 1148 1149SDValue X86TargetLowering::LowerMemArgument(SDValue Op, SelectionDAG &DAG, 1150 const CCValAssign &VA, 1151 MachineFrameInfo *MFI, 1152 unsigned CC, 1153 SDValue Root, unsigned i) { 1154 // Create the nodes corresponding to a load from this parameter slot. 1155 ISD::ArgFlagsTy Flags = 1156 cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); 1157 bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; 1158 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 1159 1160 // FIXME: For now, all byval parameter objects are marked mutable. This can be 1161 // changed with more analysis. 1162 // In case of tail call optimization mark all arguments mutable. Since they 1163 // could be overwritten by lowering of arguments in case of a tail call. 1164 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, 1165 VA.getLocMemOffset(), isImmutable); 1166 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 1167 if (Flags.isByVal()) 1168 return FIN; 1169 return DAG.getLoad(VA.getValVT(), Root, FIN, 1170 PseudoSourceValue::getFixedStack(FI), 0); 1171} 1172 1173SDValue 1174X86TargetLowering::LowerFORMAL_ARGUMENTS(SDValue Op, SelectionDAG &DAG) { 1175 MachineFunction &MF = DAG.getMachineFunction(); 1176 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1177 1178 const Function* Fn = MF.getFunction(); 1179 if (Fn->hasExternalLinkage() && 1180 Subtarget->isTargetCygMing() && 1181 Fn->getName() == "main") 1182 FuncInfo->setForceFramePointer(true); 1183 1184 // Decorate the function name. 1185 FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); 1186 1187 MachineFrameInfo *MFI = MF.getFrameInfo(); 1188 SDValue Root = Op.getOperand(0); 1189 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1190 unsigned CC = MF.getFunction()->getCallingConv(); 1191 bool Is64Bit = Subtarget->is64Bit(); 1192 bool IsWin64 = Subtarget->isTargetWin64(); 1193 1194 assert(!(isVarArg && CC == CallingConv::Fast) && 1195 "Var args not supported with calling convention fastcc"); 1196 1197 // Assign locations to all of the incoming arguments. 1198 SmallVector<CCValAssign, 16> ArgLocs; 1199 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1200 CCInfo.AnalyzeFormalArguments(Op.getNode(), CCAssignFnForNode(Op)); 1201 1202 SmallVector<SDValue, 8> ArgValues; 1203 unsigned LastVal = ~0U; 1204 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1205 CCValAssign &VA = ArgLocs[i]; 1206 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 1207 // places. 1208 assert(VA.getValNo() != LastVal && 1209 "Don't support value assigned to multiple locs yet"); 1210 LastVal = VA.getValNo(); 1211 1212 if (VA.isRegLoc()) { 1213 MVT RegVT = VA.getLocVT(); 1214 TargetRegisterClass *RC; 1215 if (RegVT == MVT::i32) 1216 RC = X86::GR32RegisterClass; 1217 else if (Is64Bit && RegVT == MVT::i64) 1218 RC = X86::GR64RegisterClass; 1219 else if (RegVT == MVT::f32) 1220 RC = X86::FR32RegisterClass; 1221 else if (RegVT == MVT::f64) 1222 RC = X86::FR64RegisterClass; 1223 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 1224 RC = X86::VR128RegisterClass; 1225 else if (RegVT.isVector()) { 1226 assert(RegVT.getSizeInBits() == 64); 1227 if (!Is64Bit) 1228 RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. 1229 else { 1230 // Darwin calling convention passes MMX values in either GPRs or 1231 // XMMs in x86-64. Other targets pass them in memory. 1232 if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { 1233 RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. 1234 RegVT = MVT::v2i64; 1235 } else { 1236 RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. 1237 RegVT = MVT::i64; 1238 } 1239 } 1240 } else { 1241 assert(0 && "Unknown argument type!"); 1242 } 1243 1244 unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); 1245 SDValue ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); 1246 1247 // If this is an 8 or 16-bit value, it is really passed promoted to 32 1248 // bits. Insert an assert[sz]ext to capture this, then truncate to the 1249 // right size. 1250 if (VA.getLocInfo() == CCValAssign::SExt) 1251 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, 1252 DAG.getValueType(VA.getValVT())); 1253 else if (VA.getLocInfo() == CCValAssign::ZExt) 1254 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, 1255 DAG.getValueType(VA.getValVT())); 1256 1257 if (VA.getLocInfo() != CCValAssign::Full) 1258 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); 1259 1260 // Handle MMX values passed in GPRs. 1261 if (Is64Bit && RegVT != VA.getLocVT()) { 1262 if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) 1263 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1264 else if (RC == X86::VR128RegisterClass) { 1265 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, 1266 DAG.getConstant(0, MVT::i64)); 1267 ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); 1268 } 1269 } 1270 1271 ArgValues.push_back(ArgValue); 1272 } else { 1273 assert(VA.isMemLoc()); 1274 ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); 1275 } 1276 } 1277 1278 // The x86-64 ABI for returning structs by value requires that we copy 1279 // the sret argument into %rax for the return. Save the argument into 1280 // a virtual register so that we can access it from the return points. 1281 if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 1282 MachineFunction &MF = DAG.getMachineFunction(); 1283 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1284 unsigned Reg = FuncInfo->getSRetReturnReg(); 1285 if (!Reg) { 1286 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 1287 FuncInfo->setSRetReturnReg(Reg); 1288 } 1289 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); 1290 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); 1291 } 1292 1293 unsigned StackSize = CCInfo.getNextStackOffset(); 1294 // align stack specially for tail calls 1295 if (PerformTailCallOpt && CC == CallingConv::Fast) 1296 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 1297 1298 // If the function takes variable number of arguments, make a frame index for 1299 // the start of the first vararg value... for expansion of llvm.va_start. 1300 if (isVarArg) { 1301 if (Is64Bit || CC != CallingConv::X86_FastCall) { 1302 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 1303 } 1304 if (Is64Bit) { 1305 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 1306 1307 // FIXME: We should really autogenerate these arrays 1308 static const unsigned GPR64ArgRegsWin64[] = { 1309 X86::RCX, X86::RDX, X86::R8, X86::R9 1310 }; 1311 static const unsigned XMMArgRegsWin64[] = { 1312 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 1313 }; 1314 static const unsigned GPR64ArgRegs64Bit[] = { 1315 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 1316 }; 1317 static const unsigned XMMArgRegs64Bit[] = { 1318 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1319 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1320 }; 1321 const unsigned *GPR64ArgRegs, *XMMArgRegs; 1322 1323 if (IsWin64) { 1324 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 1325 GPR64ArgRegs = GPR64ArgRegsWin64; 1326 XMMArgRegs = XMMArgRegsWin64; 1327 } else { 1328 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 1329 GPR64ArgRegs = GPR64ArgRegs64Bit; 1330 XMMArgRegs = XMMArgRegs64Bit; 1331 } 1332 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 1333 TotalNumIntRegs); 1334 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 1335 TotalNumXMMRegs); 1336 1337 // For X86-64, if there are vararg parameters that are passed via 1338 // registers, then we must store them to their spots on the stack so they 1339 // may be loaded by deferencing the result of va_next. 1340 VarArgsGPOffset = NumIntRegs * 8; 1341 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 1342 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 1343 TotalNumXMMRegs * 16, 16); 1344 1345 // Store the integer parameter registers. 1346 SmallVector<SDValue, 8> MemOps; 1347 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 1348 SDValue FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1349 DAG.getIntPtrConstant(VarArgsGPOffset)); 1350 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 1351 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], 1352 X86::GR64RegisterClass); 1353 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); 1354 SDValue Store = 1355 DAG.getStore(Val.getValue(1), Val, FIN, 1356 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1357 MemOps.push_back(Store); 1358 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1359 DAG.getIntPtrConstant(8)); 1360 } 1361 1362 // Now store the XMM (fp + vector) parameter registers. 1363 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, 1364 DAG.getIntPtrConstant(VarArgsFPOffset)); 1365 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 1366 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], 1367 X86::VR128RegisterClass); 1368 SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); 1369 SDValue Store = 1370 DAG.getStore(Val.getValue(1), Val, FIN, 1371 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); 1372 MemOps.push_back(Store); 1373 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, 1374 DAG.getIntPtrConstant(16)); 1375 } 1376 if (!MemOps.empty()) 1377 Root = DAG.getNode(ISD::TokenFactor, MVT::Other, 1378 &MemOps[0], MemOps.size()); 1379 } 1380 } 1381 1382 ArgValues.push_back(Root); 1383 1384 // Some CCs need callee pop. 1385 if (IsCalleePop(Op)) { 1386 BytesToPopOnReturn = StackSize; // Callee pops everything. 1387 BytesCallerReserves = 0; 1388 } else { 1389 BytesToPopOnReturn = 0; // Callee pops nothing. 1390 // If this is an sret function, the return should pop the hidden pointer. 1391 if (!Is64Bit && ArgsAreStructReturn(Op)) 1392 BytesToPopOnReturn = 4; 1393 BytesCallerReserves = StackSize; 1394 } 1395 1396 if (!Is64Bit) { 1397 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 1398 if (CC == CallingConv::X86_FastCall) 1399 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 1400 } 1401 1402 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 1403 1404 // Return the new list of results. 1405 return DAG.getMergeValues(Op.getNode()->getVTList(), &ArgValues[0], 1406 ArgValues.size()).getValue(Op.getResNo()); 1407} 1408 1409SDValue 1410X86TargetLowering::LowerMemOpCallTo(SDValue Op, SelectionDAG &DAG, 1411 const SDValue &StackPtr, 1412 const CCValAssign &VA, 1413 SDValue Chain, 1414 SDValue Arg) { 1415 unsigned LocMemOffset = VA.getLocMemOffset(); 1416 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 1417 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); 1418 ISD::ArgFlagsTy Flags = 1419 cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))->getArgFlags(); 1420 if (Flags.isByVal()) { 1421 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); 1422 } 1423 return DAG.getStore(Chain, Arg, PtrOff, 1424 PseudoSourceValue::getStack(), LocMemOffset); 1425} 1426 1427/// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call 1428/// optimization is performed and it is required. 1429SDValue 1430X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 1431 SDValue &OutRetAddr, 1432 SDValue Chain, 1433 bool IsTailCall, 1434 bool Is64Bit, 1435 int FPDiff) { 1436 if (!IsTailCall || FPDiff==0) return Chain; 1437 1438 // Adjust the Return address stack slot. 1439 MVT VT = getPointerTy(); 1440 OutRetAddr = getReturnAddressFrameIndex(DAG); 1441 // Load the "old" Return address. 1442 OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); 1443 return SDValue(OutRetAddr.getNode(), 1); 1444} 1445 1446/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 1447/// optimization is performed and it is required (FPDiff!=0). 1448static SDValue 1449EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 1450 SDValue Chain, SDValue RetAddrFrIdx, 1451 bool Is64Bit, int FPDiff) { 1452 // Store the return address to the appropriate stack slot. 1453 if (!FPDiff) return Chain; 1454 // Calculate the new stack slot for the return address. 1455 int SlotSize = Is64Bit ? 8 : 4; 1456 int NewReturnAddrFI = 1457 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 1458 MVT VT = Is64Bit ? MVT::i64 : MVT::i32; 1459 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 1460 Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, 1461 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 1462 return Chain; 1463} 1464 1465SDValue X86TargetLowering::LowerCALL(SDValue Op, SelectionDAG &DAG) { 1466 MachineFunction &MF = DAG.getMachineFunction(); 1467 SDValue Chain = Op.getOperand(0); 1468 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 1469 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; 1470 bool IsTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0 1471 && CC == CallingConv::Fast && PerformTailCallOpt; 1472 SDValue Callee = Op.getOperand(4); 1473 bool Is64Bit = Subtarget->is64Bit(); 1474 bool IsStructRet = CallIsStructReturn(Op); 1475 1476 assert(!(isVarArg && CC == CallingConv::Fast) && 1477 "Var args not supported with calling convention fastcc"); 1478 1479 // Analyze operands of the call, assigning locations to each operand. 1480 SmallVector<CCValAssign, 16> ArgLocs; 1481 CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); 1482 CCInfo.AnalyzeCallOperands(Op.getNode(), CCAssignFnForNode(Op)); 1483 1484 // Get a count of how many bytes are to be pushed on the stack. 1485 unsigned NumBytes = CCInfo.getNextStackOffset(); 1486 if (IsTailCall) 1487 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 1488 1489 int FPDiff = 0; 1490 if (IsTailCall) { 1491 // Lower arguments at fp - stackoffset + fpdiff. 1492 unsigned NumBytesCallerPushed = 1493 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 1494 FPDiff = NumBytesCallerPushed - NumBytes; 1495 1496 // Set the delta of movement of the returnaddr stackslot. 1497 // But only set if delta is greater than previous delta. 1498 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 1499 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 1500 } 1501 1502 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes)); 1503 1504 SDValue RetAddrFrIdx; 1505 // Load return adress for tail calls. 1506 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, 1507 FPDiff); 1508 1509 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 1510 SmallVector<SDValue, 8> MemOpChains; 1511 SDValue StackPtr; 1512 1513 // Walk the register/memloc assignments, inserting copies/loads. In the case 1514 // of tail call optimization arguments are handle later. 1515 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1516 CCValAssign &VA = ArgLocs[i]; 1517 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1518 bool isByVal = cast<ARG_FLAGSSDNode>(Op.getOperand(6+2*VA.getValNo()))-> 1519 getArgFlags().isByVal(); 1520 1521 // Promote the value if needed. 1522 switch (VA.getLocInfo()) { 1523 default: assert(0 && "Unknown loc info!"); 1524 case CCValAssign::Full: break; 1525 case CCValAssign::SExt: 1526 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); 1527 break; 1528 case CCValAssign::ZExt: 1529 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); 1530 break; 1531 case CCValAssign::AExt: 1532 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); 1533 break; 1534 } 1535 1536 if (VA.isRegLoc()) { 1537 if (Is64Bit) { 1538 MVT RegVT = VA.getLocVT(); 1539 if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 1540 switch (VA.getLocReg()) { 1541 default: 1542 break; 1543 case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: 1544 case X86::R8: { 1545 // Special case: passing MMX values in GPR registers. 1546 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1547 break; 1548 } 1549 case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: 1550 case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { 1551 // Special case: passing MMX values in XMM registers. 1552 Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); 1553 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); 1554 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 1555 DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, 1556 getMOVLMask(2, DAG)); 1557 break; 1558 } 1559 } 1560 } 1561 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 1562 } else { 1563 if (!IsTailCall || (IsTailCall && isByVal)) { 1564 assert(VA.isMemLoc()); 1565 if (StackPtr.getNode() == 0) 1566 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1567 1568 MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain, 1569 Arg)); 1570 } 1571 } 1572 } 1573 1574 if (!MemOpChains.empty()) 1575 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1576 &MemOpChains[0], MemOpChains.size()); 1577 1578 // Build a sequence of copy-to-reg nodes chained together with token chain 1579 // and flag operands which copy the outgoing args into registers. 1580 SDValue InFlag; 1581 // Tail call byval lowering might overwrite argument registers so in case of 1582 // tail call optimization the copies to registers are lowered later. 1583 if (!IsTailCall) 1584 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1585 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1586 InFlag); 1587 InFlag = Chain.getValue(1); 1588 } 1589 1590 // ELF / PIC requires GOT in the EBX register before function calls via PLT 1591 // GOT pointer. 1592 if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { 1593 Chain = DAG.getCopyToReg(Chain, X86::EBX, 1594 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 1595 InFlag); 1596 InFlag = Chain.getValue(1); 1597 } 1598 // If we are tail calling and generating PIC/GOT style code load the address 1599 // of the callee into ecx. The value in ecx is used as target of the tail 1600 // jump. This is done to circumvent the ebx/callee-saved problem for tail 1601 // calls on PIC/GOT architectures. Normally we would just put the address of 1602 // GOT into ebx and then call target@PLT. But for tail callss ebx would be 1603 // restored (since ebx is callee saved) before jumping to the target@PLT. 1604 if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { 1605 // Note: The actual moving to ecx is done further down. 1606 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 1607 if (G && !G->getGlobal()->hasHiddenVisibility() && 1608 !G->getGlobal()->hasProtectedVisibility()) 1609 Callee = LowerGlobalAddress(Callee, DAG); 1610 else if (isa<ExternalSymbolSDNode>(Callee)) 1611 Callee = LowerExternalSymbol(Callee,DAG); 1612 } 1613 1614 if (Is64Bit && isVarArg) { 1615 // From AMD64 ABI document: 1616 // For calls that may call functions that use varargs or stdargs 1617 // (prototype-less calls or calls to functions containing ellipsis (...) in 1618 // the declaration) %al is used as hidden argument to specify the number 1619 // of SSE registers used. The contents of %al do not need to match exactly 1620 // the number of registers, but must be an ubound on the number of SSE 1621 // registers used and is in the range 0 - 8 inclusive. 1622 1623 // FIXME: Verify this on Win64 1624 // Count the number of XMM registers allocated. 1625 static const unsigned XMMArgRegs[] = { 1626 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 1627 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 1628 }; 1629 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 1630 1631 Chain = DAG.getCopyToReg(Chain, X86::AL, 1632 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 1633 InFlag = Chain.getValue(1); 1634 } 1635 1636 1637 // For tail calls lower the arguments to the 'real' stack slot. 1638 if (IsTailCall) { 1639 SmallVector<SDValue, 8> MemOpChains2; 1640 SDValue FIN; 1641 int FI = 0; 1642 // Do not flag preceeding copytoreg stuff together with the following stuff. 1643 InFlag = SDValue(); 1644 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 1645 CCValAssign &VA = ArgLocs[i]; 1646 if (!VA.isRegLoc()) { 1647 assert(VA.isMemLoc()); 1648 SDValue Arg = Op.getOperand(5+2*VA.getValNo()); 1649 SDValue FlagsOp = Op.getOperand(6+2*VA.getValNo()); 1650 ISD::ArgFlagsTy Flags = 1651 cast<ARG_FLAGSSDNode>(FlagsOp)->getArgFlags(); 1652 // Create frame index. 1653 int32_t Offset = VA.getLocMemOffset()+FPDiff; 1654 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 1655 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 1656 FIN = DAG.getFrameIndex(FI, getPointerTy()); 1657 1658 if (Flags.isByVal()) { 1659 // Copy relative to framepointer. 1660 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 1661 if (StackPtr.getNode() == 0) 1662 StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); 1663 Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); 1664 1665 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, 1666 Flags, DAG)); 1667 } else { 1668 // Store relative to framepointer. 1669 MemOpChains2.push_back( 1670 DAG.getStore(Chain, Arg, FIN, 1671 PseudoSourceValue::getFixedStack(FI), 0)); 1672 } 1673 } 1674 } 1675 1676 if (!MemOpChains2.empty()) 1677 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, 1678 &MemOpChains2[0], MemOpChains2.size()); 1679 1680 // Copy arguments to their registers. 1681 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 1682 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, 1683 InFlag); 1684 InFlag = Chain.getValue(1); 1685 } 1686 InFlag =SDValue(); 1687 1688 // Store the return address to the appropriate stack slot. 1689 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 1690 FPDiff); 1691 } 1692 1693 // If the callee is a GlobalAddress node (quite common, every direct call is) 1694 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 1695 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1696 // We should use extra load for direct calls to dllimported functions in 1697 // non-JIT mode. 1698 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), 1699 getTargetMachine(), true)) 1700 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 1701 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1702 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 1703 } else if (IsTailCall) { 1704 unsigned Opc = Is64Bit ? X86::R9 : X86::ECX; 1705 1706 Chain = DAG.getCopyToReg(Chain, 1707 DAG.getRegister(Opc, getPointerTy()), 1708 Callee,InFlag); 1709 Callee = DAG.getRegister(Opc, getPointerTy()); 1710 // Add register as live out. 1711 DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); 1712 } 1713 1714 // Returns a chain & a flag for retval copy to use. 1715 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1716 SmallVector<SDValue, 8> Ops; 1717 1718 if (IsTailCall) { 1719 Ops.push_back(Chain); 1720 Ops.push_back(DAG.getIntPtrConstant(NumBytes)); 1721 Ops.push_back(DAG.getIntPtrConstant(0)); 1722 if (InFlag.getNode()) 1723 Ops.push_back(InFlag); 1724 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); 1725 InFlag = Chain.getValue(1); 1726 1727 // Returns a chain & a flag for retval copy to use. 1728 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 1729 Ops.clear(); 1730 } 1731 1732 Ops.push_back(Chain); 1733 Ops.push_back(Callee); 1734 1735 if (IsTailCall) 1736 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 1737 1738 // Add argument registers to the end of the list so that they are known live 1739 // into the call. 1740 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 1741 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 1742 RegsToPass[i].second.getValueType())); 1743 1744 // Add an implicit use GOT pointer in EBX. 1745 if (!IsTailCall && !Is64Bit && 1746 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 1747 Subtarget->isPICStyleGOT()) 1748 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 1749 1750 // Add an implicit use of AL for x86 vararg functions. 1751 if (Is64Bit && isVarArg) 1752 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 1753 1754 if (InFlag.getNode()) 1755 Ops.push_back(InFlag); 1756 1757 if (IsTailCall) { 1758 assert(InFlag.getNode() && 1759 "Flag must be set. Depend on flag being set in LowerRET"); 1760 Chain = DAG.getNode(X86ISD::TAILCALL, 1761 Op.getNode()->getVTList(), &Ops[0], Ops.size()); 1762 1763 return SDValue(Chain.getNode(), Op.getResNo()); 1764 } 1765 1766 Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); 1767 InFlag = Chain.getValue(1); 1768 1769 // Create the CALLSEQ_END node. 1770 unsigned NumBytesForCalleeToPush; 1771 if (IsCalleePop(Op)) 1772 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 1773 else if (!Is64Bit && IsStructRet) 1774 // If this is is a call to a struct-return function, the callee 1775 // pops the hidden struct pointer, so we have to push it back. 1776 // This is common for Darwin/X86, Linux & Mingw32 targets. 1777 NumBytesForCalleeToPush = 4; 1778 else 1779 NumBytesForCalleeToPush = 0; // Callee pops nothing. 1780 1781 // Returns a flag for retval copy to use. 1782 Chain = DAG.getCALLSEQ_END(Chain, 1783 DAG.getIntPtrConstant(NumBytes), 1784 DAG.getIntPtrConstant(NumBytesForCalleeToPush), 1785 InFlag); 1786 InFlag = Chain.getValue(1); 1787 1788 // Handle result values, copying them out of physregs into vregs that we 1789 // return. 1790 return SDValue(LowerCallResult(Chain, InFlag, Op.getNode(), CC, DAG), 1791 Op.getResNo()); 1792} 1793 1794 1795//===----------------------------------------------------------------------===// 1796// Fast Calling Convention (tail call) implementation 1797//===----------------------------------------------------------------------===// 1798 1799// Like std call, callee cleans arguments, convention except that ECX is 1800// reserved for storing the tail called function address. Only 2 registers are 1801// free for argument passing (inreg). Tail call optimization is performed 1802// provided: 1803// * tailcallopt is enabled 1804// * caller/callee are fastcc 1805// On X86_64 architecture with GOT-style position independent code only local 1806// (within module) calls are supported at the moment. 1807// To keep the stack aligned according to platform abi the function 1808// GetAlignedArgumentStackSize ensures that argument delta is always multiples 1809// of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 1810// If a tail called function callee has more arguments than the caller the 1811// caller needs to make sure that there is room to move the RETADDR to. This is 1812// achieved by reserving an area the size of the argument delta right after the 1813// original REtADDR, but before the saved framepointer or the spilled registers 1814// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 1815// stack layout: 1816// arg1 1817// arg2 1818// RETADDR 1819// [ new RETADDR 1820// move area ] 1821// (possible EBP) 1822// ESI 1823// EDI 1824// local1 .. 1825 1826/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 1827/// for a 16 byte align requirement. 1828unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 1829 SelectionDAG& DAG) { 1830 MachineFunction &MF = DAG.getMachineFunction(); 1831 const TargetMachine &TM = MF.getTarget(); 1832 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 1833 unsigned StackAlignment = TFI.getStackAlignment(); 1834 uint64_t AlignMask = StackAlignment - 1; 1835 int64_t Offset = StackSize; 1836 unsigned SlotSize = Subtarget->is64Bit() ? 8 : 4; 1837 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 1838 // Number smaller than 12 so just add the difference. 1839 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 1840 } else { 1841 // Mask out lower bits, add stackalignment once plus the 12 bytes. 1842 Offset = ((~AlignMask) & Offset) + StackAlignment + 1843 (StackAlignment-SlotSize); 1844 } 1845 return Offset; 1846} 1847 1848/// IsEligibleForTailCallElimination - Check to see whether the next instruction 1849/// following the call is a return. A function is eligible if caller/callee 1850/// calling conventions match, currently only fastcc supports tail calls, and 1851/// the function CALL is immediatly followed by a RET. 1852bool X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Call, 1853 SDValue Ret, 1854 SelectionDAG& DAG) const { 1855 if (!PerformTailCallOpt) 1856 return false; 1857 1858 if (CheckTailCallReturnConstraints(Call, Ret)) { 1859 MachineFunction &MF = DAG.getMachineFunction(); 1860 unsigned CallerCC = MF.getFunction()->getCallingConv(); 1861 unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue(); 1862 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 1863 SDValue Callee = Call.getOperand(4); 1864 // On x86/32Bit PIC/GOT tail calls are supported. 1865 if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || 1866 !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) 1867 return true; 1868 1869 // Can only do local tail calls (in same module, hidden or protected) on 1870 // x86_64 PIC/GOT at the moment. 1871 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1872 return G->getGlobal()->hasHiddenVisibility() 1873 || G->getGlobal()->hasProtectedVisibility(); 1874 } 1875 } 1876 1877 return false; 1878} 1879 1880FastISel * 1881X86TargetLowering::createFastISel(MachineFunction &mf, 1882 DenseMap<const Value *, unsigned> &vm, 1883 DenseMap<const BasicBlock *, 1884 MachineBasicBlock *> &bm) { 1885 return X86::createFastISel(mf, vm, bm); 1886} 1887 1888 1889//===----------------------------------------------------------------------===// 1890// Other Lowering Hooks 1891//===----------------------------------------------------------------------===// 1892 1893 1894SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1895 MachineFunction &MF = DAG.getMachineFunction(); 1896 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 1897 int ReturnAddrIndex = FuncInfo->getRAIndex(); 1898 1899 if (ReturnAddrIndex == 0) { 1900 // Set up a frame object for the return address. 1901 if (Subtarget->is64Bit()) 1902 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); 1903 else 1904 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1905 1906 FuncInfo->setRAIndex(ReturnAddrIndex); 1907 } 1908 1909 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 1910} 1911 1912 1913/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 1914/// specific condition code. It returns a false if it cannot do a direct 1915/// translation. X86CC is the translated CondCode. LHS/RHS are modified as 1916/// needed. 1917static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 1918 unsigned &X86CC, SDValue &LHS, SDValue &RHS, 1919 SelectionDAG &DAG) { 1920 X86CC = X86::COND_INVALID; 1921 if (!isFP) { 1922 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 1923 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 1924 // X > -1 -> X == 0, jump !sign. 1925 RHS = DAG.getConstant(0, RHS.getValueType()); 1926 X86CC = X86::COND_NS; 1927 return true; 1928 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 1929 // X < 0 -> X == 0, jump on sign. 1930 X86CC = X86::COND_S; 1931 return true; 1932 } else if (SetCCOpcode == ISD::SETLT && RHSC->getValue() == 1) { 1933 // X < 1 -> X <= 0 1934 RHS = DAG.getConstant(0, RHS.getValueType()); 1935 X86CC = X86::COND_LE; 1936 return true; 1937 } 1938 } 1939 1940 switch (SetCCOpcode) { 1941 default: break; 1942 case ISD::SETEQ: X86CC = X86::COND_E; break; 1943 case ISD::SETGT: X86CC = X86::COND_G; break; 1944 case ISD::SETGE: X86CC = X86::COND_GE; break; 1945 case ISD::SETLT: X86CC = X86::COND_L; break; 1946 case ISD::SETLE: X86CC = X86::COND_LE; break; 1947 case ISD::SETNE: X86CC = X86::COND_NE; break; 1948 case ISD::SETULT: X86CC = X86::COND_B; break; 1949 case ISD::SETUGT: X86CC = X86::COND_A; break; 1950 case ISD::SETULE: X86CC = X86::COND_BE; break; 1951 case ISD::SETUGE: X86CC = X86::COND_AE; break; 1952 } 1953 } else { 1954 // First determine if it requires or is profitable to flip the operands. 1955 bool Flip = false; 1956 switch (SetCCOpcode) { 1957 default: break; 1958 case ISD::SETOLT: 1959 case ISD::SETOLE: 1960 case ISD::SETUGT: 1961 case ISD::SETUGE: 1962 Flip = true; 1963 break; 1964 } 1965 1966 // If LHS is a foldable load, but RHS is not, flip the condition. 1967 if (!Flip && 1968 (ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) && 1969 !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) { 1970 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); 1971 Flip = true; 1972 } 1973 if (Flip) 1974 std::swap(LHS, RHS); 1975 1976 // On a floating point condition, the flags are set as follows: 1977 // ZF PF CF op 1978 // 0 | 0 | 0 | X > Y 1979 // 0 | 0 | 1 | X < Y 1980 // 1 | 0 | 0 | X == Y 1981 // 1 | 1 | 1 | unordered 1982 switch (SetCCOpcode) { 1983 default: break; 1984 case ISD::SETUEQ: 1985 case ISD::SETEQ: 1986 X86CC = X86::COND_E; 1987 break; 1988 case ISD::SETOLT: // flipped 1989 case ISD::SETOGT: 1990 case ISD::SETGT: 1991 X86CC = X86::COND_A; 1992 break; 1993 case ISD::SETOLE: // flipped 1994 case ISD::SETOGE: 1995 case ISD::SETGE: 1996 X86CC = X86::COND_AE; 1997 break; 1998 case ISD::SETUGT: // flipped 1999 case ISD::SETULT: 2000 case ISD::SETLT: 2001 X86CC = X86::COND_B; 2002 break; 2003 case ISD::SETUGE: // flipped 2004 case ISD::SETULE: 2005 case ISD::SETLE: 2006 X86CC = X86::COND_BE; 2007 break; 2008 case ISD::SETONE: 2009 case ISD::SETNE: 2010 X86CC = X86::COND_NE; 2011 break; 2012 case ISD::SETUO: 2013 X86CC = X86::COND_P; 2014 break; 2015 case ISD::SETO: 2016 X86CC = X86::COND_NP; 2017 break; 2018 } 2019 } 2020 2021 return X86CC != X86::COND_INVALID; 2022} 2023 2024/// hasFPCMov - is there a floating point cmov for the specific X86 condition 2025/// code. Current x86 isa includes the following FP cmov instructions: 2026/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 2027static bool hasFPCMov(unsigned X86CC) { 2028 switch (X86CC) { 2029 default: 2030 return false; 2031 case X86::COND_B: 2032 case X86::COND_BE: 2033 case X86::COND_E: 2034 case X86::COND_P: 2035 case X86::COND_A: 2036 case X86::COND_AE: 2037 case X86::COND_NE: 2038 case X86::COND_NP: 2039 return true; 2040 } 2041} 2042 2043/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return 2044/// true if Op is undef or if its value falls within the specified range (L, H]. 2045static bool isUndefOrInRange(SDValue Op, unsigned Low, unsigned Hi) { 2046 if (Op.getOpcode() == ISD::UNDEF) 2047 return true; 2048 2049 unsigned Val = cast<ConstantSDNode>(Op)->getValue(); 2050 return (Val >= Low && Val < Hi); 2051} 2052 2053/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return 2054/// true if Op is undef or if its value equal to the specified value. 2055static bool isUndefOrEqual(SDValue Op, unsigned Val) { 2056 if (Op.getOpcode() == ISD::UNDEF) 2057 return true; 2058 return cast<ConstantSDNode>(Op)->getValue() == Val; 2059} 2060 2061/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand 2062/// specifies a shuffle of elements that is suitable for input to PSHUFD. 2063bool X86::isPSHUFDMask(SDNode *N) { 2064 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2065 2066 if (N->getNumOperands() != 2 && N->getNumOperands() != 4) 2067 return false; 2068 2069 // Check if the value doesn't reference the second vector. 2070 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2071 SDValue Arg = N->getOperand(i); 2072 if (Arg.getOpcode() == ISD::UNDEF) continue; 2073 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2074 if (cast<ConstantSDNode>(Arg)->getValue() >= e) 2075 return false; 2076 } 2077 2078 return true; 2079} 2080 2081/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand 2082/// specifies a shuffle of elements that is suitable for input to PSHUFHW. 2083bool X86::isPSHUFHWMask(SDNode *N) { 2084 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2085 2086 if (N->getNumOperands() != 8) 2087 return false; 2088 2089 // Lower quadword copied in order. 2090 for (unsigned i = 0; i != 4; ++i) { 2091 SDValue Arg = N->getOperand(i); 2092 if (Arg.getOpcode() == ISD::UNDEF) continue; 2093 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2094 if (cast<ConstantSDNode>(Arg)->getValue() != i) 2095 return false; 2096 } 2097 2098 // Upper quadword shuffled. 2099 for (unsigned i = 4; i != 8; ++i) { 2100 SDValue Arg = N->getOperand(i); 2101 if (Arg.getOpcode() == ISD::UNDEF) continue; 2102 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2103 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2104 if (Val < 4 || Val > 7) 2105 return false; 2106 } 2107 2108 return true; 2109} 2110 2111/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand 2112/// specifies a shuffle of elements that is suitable for input to PSHUFLW. 2113bool X86::isPSHUFLWMask(SDNode *N) { 2114 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2115 2116 if (N->getNumOperands() != 8) 2117 return false; 2118 2119 // Upper quadword copied in order. 2120 for (unsigned i = 4; i != 8; ++i) 2121 if (!isUndefOrEqual(N->getOperand(i), i)) 2122 return false; 2123 2124 // Lower quadword shuffled. 2125 for (unsigned i = 0; i != 4; ++i) 2126 if (!isUndefOrInRange(N->getOperand(i), 0, 4)) 2127 return false; 2128 2129 return true; 2130} 2131 2132/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 2133/// specifies a shuffle of elements that is suitable for input to SHUFP*. 2134static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { 2135 if (NumElems != 2 && NumElems != 4) return false; 2136 2137 unsigned Half = NumElems / 2; 2138 for (unsigned i = 0; i < Half; ++i) 2139 if (!isUndefOrInRange(Elems[i], 0, NumElems)) 2140 return false; 2141 for (unsigned i = Half; i < NumElems; ++i) 2142 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) 2143 return false; 2144 2145 return true; 2146} 2147 2148bool X86::isSHUFPMask(SDNode *N) { 2149 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2150 return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); 2151} 2152 2153/// isCommutedSHUFP - Returns true if the shuffle mask is exactly 2154/// the reverse of what x86 shuffles want. x86 shuffles requires the lower 2155/// half elements to come from vector 1 (which would equal the dest.) and 2156/// the upper half to come from vector 2. 2157static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { 2158 if (NumOps != 2 && NumOps != 4) return false; 2159 2160 unsigned Half = NumOps / 2; 2161 for (unsigned i = 0; i < Half; ++i) 2162 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) 2163 return false; 2164 for (unsigned i = Half; i < NumOps; ++i) 2165 if (!isUndefOrInRange(Ops[i], 0, NumOps)) 2166 return false; 2167 return true; 2168} 2169 2170static bool isCommutedSHUFP(SDNode *N) { 2171 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2172 return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); 2173} 2174 2175/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 2176/// specifies a shuffle of elements that is suitable for input to MOVHLPS. 2177bool X86::isMOVHLPSMask(SDNode *N) { 2178 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2179 2180 if (N->getNumOperands() != 4) 2181 return false; 2182 2183 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 2184 return isUndefOrEqual(N->getOperand(0), 6) && 2185 isUndefOrEqual(N->getOperand(1), 7) && 2186 isUndefOrEqual(N->getOperand(2), 2) && 2187 isUndefOrEqual(N->getOperand(3), 3); 2188} 2189 2190/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 2191/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 2192/// <2, 3, 2, 3> 2193bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { 2194 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2195 2196 if (N->getNumOperands() != 4) 2197 return false; 2198 2199 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 2200 return isUndefOrEqual(N->getOperand(0), 2) && 2201 isUndefOrEqual(N->getOperand(1), 3) && 2202 isUndefOrEqual(N->getOperand(2), 2) && 2203 isUndefOrEqual(N->getOperand(3), 3); 2204} 2205 2206/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 2207/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 2208bool X86::isMOVLPMask(SDNode *N) { 2209 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2210 2211 unsigned NumElems = N->getNumOperands(); 2212 if (NumElems != 2 && NumElems != 4) 2213 return false; 2214 2215 for (unsigned i = 0; i < NumElems/2; ++i) 2216 if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) 2217 return false; 2218 2219 for (unsigned i = NumElems/2; i < NumElems; ++i) 2220 if (!isUndefOrEqual(N->getOperand(i), i)) 2221 return false; 2222 2223 return true; 2224} 2225 2226/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 2227/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 2228/// and MOVLHPS. 2229bool X86::isMOVHPMask(SDNode *N) { 2230 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2231 2232 unsigned NumElems = N->getNumOperands(); 2233 if (NumElems != 2 && NumElems != 4) 2234 return false; 2235 2236 for (unsigned i = 0; i < NumElems/2; ++i) 2237 if (!isUndefOrEqual(N->getOperand(i), i)) 2238 return false; 2239 2240 for (unsigned i = 0; i < NumElems/2; ++i) { 2241 SDValue Arg = N->getOperand(i + NumElems/2); 2242 if (!isUndefOrEqual(Arg, i + NumElems)) 2243 return false; 2244 } 2245 2246 return true; 2247} 2248 2249/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 2250/// specifies a shuffle of elements that is suitable for input to UNPCKL. 2251bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, 2252 bool V2IsSplat = false) { 2253 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2254 return false; 2255 2256 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2257 SDValue BitI = Elts[i]; 2258 SDValue BitI1 = Elts[i+1]; 2259 if (!isUndefOrEqual(BitI, j)) 2260 return false; 2261 if (V2IsSplat) { 2262 if (isUndefOrEqual(BitI1, NumElts)) 2263 return false; 2264 } else { 2265 if (!isUndefOrEqual(BitI1, j + NumElts)) 2266 return false; 2267 } 2268 } 2269 2270 return true; 2271} 2272 2273bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { 2274 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2275 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2276} 2277 2278/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 2279/// specifies a shuffle of elements that is suitable for input to UNPCKH. 2280bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, 2281 bool V2IsSplat = false) { 2282 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 2283 return false; 2284 2285 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { 2286 SDValue BitI = Elts[i]; 2287 SDValue BitI1 = Elts[i+1]; 2288 if (!isUndefOrEqual(BitI, j + NumElts/2)) 2289 return false; 2290 if (V2IsSplat) { 2291 if (isUndefOrEqual(BitI1, NumElts)) 2292 return false; 2293 } else { 2294 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 2295 return false; 2296 } 2297 } 2298 2299 return true; 2300} 2301 2302bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { 2303 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2304 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); 2305} 2306 2307/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 2308/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 2309/// <0, 0, 1, 1> 2310bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { 2311 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2312 2313 unsigned NumElems = N->getNumOperands(); 2314 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2315 return false; 2316 2317 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { 2318 SDValue BitI = N->getOperand(i); 2319 SDValue BitI1 = N->getOperand(i+1); 2320 2321 if (!isUndefOrEqual(BitI, j)) 2322 return false; 2323 if (!isUndefOrEqual(BitI1, j)) 2324 return false; 2325 } 2326 2327 return true; 2328} 2329 2330/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 2331/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 2332/// <2, 2, 3, 3> 2333bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { 2334 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2335 2336 unsigned NumElems = N->getNumOperands(); 2337 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 2338 return false; 2339 2340 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 2341 SDValue BitI = N->getOperand(i); 2342 SDValue BitI1 = N->getOperand(i + 1); 2343 2344 if (!isUndefOrEqual(BitI, j)) 2345 return false; 2346 if (!isUndefOrEqual(BitI1, j)) 2347 return false; 2348 } 2349 2350 return true; 2351} 2352 2353/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 2354/// specifies a shuffle of elements that is suitable for input to MOVSS, 2355/// MOVSD, and MOVD, i.e. setting the lowest element. 2356static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { 2357 if (NumElts != 2 && NumElts != 4) 2358 return false; 2359 2360 if (!isUndefOrEqual(Elts[0], NumElts)) 2361 return false; 2362 2363 for (unsigned i = 1; i < NumElts; ++i) { 2364 if (!isUndefOrEqual(Elts[i], i)) 2365 return false; 2366 } 2367 2368 return true; 2369} 2370 2371bool X86::isMOVLMask(SDNode *N) { 2372 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2373 return ::isMOVLMask(N->op_begin(), N->getNumOperands()); 2374} 2375 2376/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 2377/// of what x86 movss want. X86 movs requires the lowest element to be lowest 2378/// element of vector 2 and the other elements to come from vector 1 in order. 2379static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, 2380 bool V2IsSplat = false, 2381 bool V2IsUndef = false) { 2382 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 2383 return false; 2384 2385 if (!isUndefOrEqual(Ops[0], 0)) 2386 return false; 2387 2388 for (unsigned i = 1; i < NumOps; ++i) { 2389 SDValue Arg = Ops[i]; 2390 if (!(isUndefOrEqual(Arg, i+NumOps) || 2391 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || 2392 (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) 2393 return false; 2394 } 2395 2396 return true; 2397} 2398 2399static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, 2400 bool V2IsUndef = false) { 2401 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2402 return isCommutedMOVL(N->op_begin(), N->getNumOperands(), 2403 V2IsSplat, V2IsUndef); 2404} 2405 2406/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2407/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 2408bool X86::isMOVSHDUPMask(SDNode *N) { 2409 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2410 2411 if (N->getNumOperands() != 4) 2412 return false; 2413 2414 // Expect 1, 1, 3, 3 2415 for (unsigned i = 0; i < 2; ++i) { 2416 SDValue Arg = N->getOperand(i); 2417 if (Arg.getOpcode() == ISD::UNDEF) continue; 2418 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2419 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2420 if (Val != 1) return false; 2421 } 2422 2423 bool HasHi = false; 2424 for (unsigned i = 2; i < 4; ++i) { 2425 SDValue Arg = N->getOperand(i); 2426 if (Arg.getOpcode() == ISD::UNDEF) continue; 2427 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2428 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2429 if (Val != 3) return false; 2430 HasHi = true; 2431 } 2432 2433 // Don't use movshdup if it can be done with a shufps. 2434 return HasHi; 2435} 2436 2437/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 2438/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 2439bool X86::isMOVSLDUPMask(SDNode *N) { 2440 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2441 2442 if (N->getNumOperands() != 4) 2443 return false; 2444 2445 // Expect 0, 0, 2, 2 2446 for (unsigned i = 0; i < 2; ++i) { 2447 SDValue Arg = N->getOperand(i); 2448 if (Arg.getOpcode() == ISD::UNDEF) continue; 2449 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2450 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2451 if (Val != 0) return false; 2452 } 2453 2454 bool HasHi = false; 2455 for (unsigned i = 2; i < 4; ++i) { 2456 SDValue Arg = N->getOperand(i); 2457 if (Arg.getOpcode() == ISD::UNDEF) continue; 2458 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2459 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2460 if (Val != 2) return false; 2461 HasHi = true; 2462 } 2463 2464 // Don't use movshdup if it can be done with a shufps. 2465 return HasHi; 2466} 2467 2468/// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand 2469/// specifies a identity operation on the LHS or RHS. 2470static bool isIdentityMask(SDNode *N, bool RHS = false) { 2471 unsigned NumElems = N->getNumOperands(); 2472 for (unsigned i = 0; i < NumElems; ++i) 2473 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) 2474 return false; 2475 return true; 2476} 2477 2478/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2479/// a splat of a single element. 2480static bool isSplatMask(SDNode *N) { 2481 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2482 2483 // This is a splat operation if each element of the permute is the same, and 2484 // if the value doesn't reference the second vector. 2485 unsigned NumElems = N->getNumOperands(); 2486 SDValue ElementBase; 2487 unsigned i = 0; 2488 for (; i != NumElems; ++i) { 2489 SDValue Elt = N->getOperand(i); 2490 if (isa<ConstantSDNode>(Elt)) { 2491 ElementBase = Elt; 2492 break; 2493 } 2494 } 2495 2496 if (!ElementBase.getNode()) 2497 return false; 2498 2499 for (; i != NumElems; ++i) { 2500 SDValue Arg = N->getOperand(i); 2501 if (Arg.getOpcode() == ISD::UNDEF) continue; 2502 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2503 if (Arg != ElementBase) return false; 2504 } 2505 2506 // Make sure it is a splat of the first vector operand. 2507 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; 2508} 2509 2510/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies 2511/// a splat of a single element and it's a 2 or 4 element mask. 2512bool X86::isSplatMask(SDNode *N) { 2513 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2514 2515 // We can only splat 64-bit, and 32-bit quantities with a single instruction. 2516 if (N->getNumOperands() != 4 && N->getNumOperands() != 2) 2517 return false; 2518 return ::isSplatMask(N); 2519} 2520 2521/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand 2522/// specifies a splat of zero element. 2523bool X86::isSplatLoMask(SDNode *N) { 2524 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2525 2526 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 2527 if (!isUndefOrEqual(N->getOperand(i), 0)) 2528 return false; 2529 return true; 2530} 2531 2532/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 2533/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 2534/// instructions. 2535unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 2536 unsigned NumOperands = N->getNumOperands(); 2537 unsigned Shift = (NumOperands == 4) ? 2 : 1; 2538 unsigned Mask = 0; 2539 for (unsigned i = 0; i < NumOperands; ++i) { 2540 unsigned Val = 0; 2541 SDValue Arg = N->getOperand(NumOperands-i-1); 2542 if (Arg.getOpcode() != ISD::UNDEF) 2543 Val = cast<ConstantSDNode>(Arg)->getValue(); 2544 if (Val >= NumOperands) Val -= NumOperands; 2545 Mask |= Val; 2546 if (i != NumOperands - 1) 2547 Mask <<= Shift; 2548 } 2549 2550 return Mask; 2551} 2552 2553/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 2554/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 2555/// instructions. 2556unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 2557 unsigned Mask = 0; 2558 // 8 nodes, but we only care about the last 4. 2559 for (unsigned i = 7; i >= 4; --i) { 2560 unsigned Val = 0; 2561 SDValue Arg = N->getOperand(i); 2562 if (Arg.getOpcode() != ISD::UNDEF) 2563 Val = cast<ConstantSDNode>(Arg)->getValue(); 2564 Mask |= (Val - 4); 2565 if (i != 4) 2566 Mask <<= 2; 2567 } 2568 2569 return Mask; 2570} 2571 2572/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 2573/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 2574/// instructions. 2575unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 2576 unsigned Mask = 0; 2577 // 8 nodes, but we only care about the first 4. 2578 for (int i = 3; i >= 0; --i) { 2579 unsigned Val = 0; 2580 SDValue Arg = N->getOperand(i); 2581 if (Arg.getOpcode() != ISD::UNDEF) 2582 Val = cast<ConstantSDNode>(Arg)->getValue(); 2583 Mask |= Val; 2584 if (i != 0) 2585 Mask <<= 2; 2586 } 2587 2588 return Mask; 2589} 2590 2591/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand 2592/// specifies a 8 element shuffle that can be broken into a pair of 2593/// PSHUFHW and PSHUFLW. 2594static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { 2595 assert(N->getOpcode() == ISD::BUILD_VECTOR); 2596 2597 if (N->getNumOperands() != 8) 2598 return false; 2599 2600 // Lower quadword shuffled. 2601 for (unsigned i = 0; i != 4; ++i) { 2602 SDValue Arg = N->getOperand(i); 2603 if (Arg.getOpcode() == ISD::UNDEF) continue; 2604 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2605 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2606 if (Val >= 4) 2607 return false; 2608 } 2609 2610 // Upper quadword shuffled. 2611 for (unsigned i = 4; i != 8; ++i) { 2612 SDValue Arg = N->getOperand(i); 2613 if (Arg.getOpcode() == ISD::UNDEF) continue; 2614 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2615 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2616 if (Val < 4 || Val > 7) 2617 return false; 2618 } 2619 2620 return true; 2621} 2622 2623/// CommuteVectorShuffle - Swap vector_shuffle operands as well as 2624/// values in ther permute mask. 2625static SDValue CommuteVectorShuffle(SDValue Op, SDValue &V1, 2626 SDValue &V2, SDValue &Mask, 2627 SelectionDAG &DAG) { 2628 MVT VT = Op.getValueType(); 2629 MVT MaskVT = Mask.getValueType(); 2630 MVT EltVT = MaskVT.getVectorElementType(); 2631 unsigned NumElems = Mask.getNumOperands(); 2632 SmallVector<SDValue, 8> MaskVec; 2633 2634 for (unsigned i = 0; i != NumElems; ++i) { 2635 SDValue Arg = Mask.getOperand(i); 2636 if (Arg.getOpcode() == ISD::UNDEF) { 2637 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2638 continue; 2639 } 2640 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2641 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2642 if (Val < NumElems) 2643 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2644 else 2645 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2646 } 2647 2648 std::swap(V1, V2); 2649 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2650 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2651} 2652 2653/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 2654/// the two vector operands have swapped position. 2655static 2656SDValue CommuteVectorShuffleMask(SDValue Mask, SelectionDAG &DAG) { 2657 MVT MaskVT = Mask.getValueType(); 2658 MVT EltVT = MaskVT.getVectorElementType(); 2659 unsigned NumElems = Mask.getNumOperands(); 2660 SmallVector<SDValue, 8> MaskVec; 2661 for (unsigned i = 0; i != NumElems; ++i) { 2662 SDValue Arg = Mask.getOperand(i); 2663 if (Arg.getOpcode() == ISD::UNDEF) { 2664 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 2665 continue; 2666 } 2667 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 2668 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2669 if (Val < NumElems) 2670 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); 2671 else 2672 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); 2673 } 2674 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); 2675} 2676 2677 2678/// ShouldXformToMOVHLPS - Return true if the node should be transformed to 2679/// match movhlps. The lower half elements should come from upper half of 2680/// V1 (and in order), and the upper half elements should come from the upper 2681/// half of V2 (and in order). 2682static bool ShouldXformToMOVHLPS(SDNode *Mask) { 2683 unsigned NumElems = Mask->getNumOperands(); 2684 if (NumElems != 4) 2685 return false; 2686 for (unsigned i = 0, e = 2; i != e; ++i) 2687 if (!isUndefOrEqual(Mask->getOperand(i), i+2)) 2688 return false; 2689 for (unsigned i = 2; i != 4; ++i) 2690 if (!isUndefOrEqual(Mask->getOperand(i), i+4)) 2691 return false; 2692 return true; 2693} 2694 2695/// isScalarLoadToVector - Returns true if the node is a scalar load that 2696/// is promoted to a vector. It also returns the LoadSDNode by reference if 2697/// required. 2698static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 2699 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 2700 N = N->getOperand(0).getNode(); 2701 if (ISD::isNON_EXTLoad(N)) { 2702 if (LD) 2703 *LD = cast<LoadSDNode>(N); 2704 return true; 2705 } 2706 } 2707 return false; 2708} 2709 2710/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 2711/// match movlp{s|d}. The lower half elements should come from lower half of 2712/// V1 (and in order), and the upper half elements should come from the upper 2713/// half of V2 (and in order). And since V1 will become the source of the 2714/// MOVLP, it must be either a vector load or a scalar load to vector. 2715static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { 2716 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 2717 return false; 2718 // Is V2 is a vector load, don't do this transformation. We will try to use 2719 // load folding shufps op. 2720 if (ISD::isNON_EXTLoad(V2)) 2721 return false; 2722 2723 unsigned NumElems = Mask->getNumOperands(); 2724 if (NumElems != 2 && NumElems != 4) 2725 return false; 2726 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 2727 if (!isUndefOrEqual(Mask->getOperand(i), i)) 2728 return false; 2729 for (unsigned i = NumElems/2; i != NumElems; ++i) 2730 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) 2731 return false; 2732 return true; 2733} 2734 2735/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 2736/// all the same. 2737static bool isSplatVector(SDNode *N) { 2738 if (N->getOpcode() != ISD::BUILD_VECTOR) 2739 return false; 2740 2741 SDValue SplatValue = N->getOperand(0); 2742 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 2743 if (N->getOperand(i) != SplatValue) 2744 return false; 2745 return true; 2746} 2747 2748/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2749/// to an undef. 2750static bool isUndefShuffle(SDNode *N) { 2751 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2752 return false; 2753 2754 SDValue V1 = N->getOperand(0); 2755 SDValue V2 = N->getOperand(1); 2756 SDValue Mask = N->getOperand(2); 2757 unsigned NumElems = Mask.getNumOperands(); 2758 for (unsigned i = 0; i != NumElems; ++i) { 2759 SDValue Arg = Mask.getOperand(i); 2760 if (Arg.getOpcode() != ISD::UNDEF) { 2761 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2762 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) 2763 return false; 2764 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) 2765 return false; 2766 } 2767 } 2768 return true; 2769} 2770 2771/// isZeroNode - Returns true if Elt is a constant zero or a floating point 2772/// constant +0.0. 2773static inline bool isZeroNode(SDValue Elt) { 2774 return ((isa<ConstantSDNode>(Elt) && 2775 cast<ConstantSDNode>(Elt)->getValue() == 0) || 2776 (isa<ConstantFPSDNode>(Elt) && 2777 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 2778} 2779 2780/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 2781/// to an zero vector. 2782static bool isZeroShuffle(SDNode *N) { 2783 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) 2784 return false; 2785 2786 SDValue V1 = N->getOperand(0); 2787 SDValue V2 = N->getOperand(1); 2788 SDValue Mask = N->getOperand(2); 2789 unsigned NumElems = Mask.getNumOperands(); 2790 for (unsigned i = 0; i != NumElems; ++i) { 2791 SDValue Arg = Mask.getOperand(i); 2792 if (Arg.getOpcode() == ISD::UNDEF) 2793 continue; 2794 2795 unsigned Idx = cast<ConstantSDNode>(Arg)->getValue(); 2796 if (Idx < NumElems) { 2797 unsigned Opc = V1.getNode()->getOpcode(); 2798 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) 2799 continue; 2800 if (Opc != ISD::BUILD_VECTOR || 2801 !isZeroNode(V1.getNode()->getOperand(Idx))) 2802 return false; 2803 } else if (Idx >= NumElems) { 2804 unsigned Opc = V2.getNode()->getOpcode(); 2805 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) 2806 continue; 2807 if (Opc != ISD::BUILD_VECTOR || 2808 !isZeroNode(V2.getNode()->getOperand(Idx - NumElems))) 2809 return false; 2810 } 2811 } 2812 return true; 2813} 2814 2815/// getZeroVector - Returns a vector of specified type with all zero elements. 2816/// 2817static SDValue getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { 2818 assert(VT.isVector() && "Expected a vector type"); 2819 2820 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2821 // type. This ensures they get CSE'd. 2822 SDValue Vec; 2823 if (VT.getSizeInBits() == 64) { // MMX 2824 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2825 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2826 } else if (HasSSE2) { // SSE2 2827 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 2828 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2829 } else { // SSE1 2830 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 2831 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); 2832 } 2833 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2834} 2835 2836/// getOnesVector - Returns a vector of specified type with all bits set. 2837/// 2838static SDValue getOnesVector(MVT VT, SelectionDAG &DAG) { 2839 assert(VT.isVector() && "Expected a vector type"); 2840 2841 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 2842 // type. This ensures they get CSE'd. 2843 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 2844 SDValue Vec; 2845 if (VT.getSizeInBits() == 64) // MMX 2846 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); 2847 else // SSE 2848 Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); 2849 return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); 2850} 2851 2852 2853/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 2854/// that point to V2 points to its first element. 2855static SDValue NormalizeMask(SDValue Mask, SelectionDAG &DAG) { 2856 assert(Mask.getOpcode() == ISD::BUILD_VECTOR); 2857 2858 bool Changed = false; 2859 SmallVector<SDValue, 8> MaskVec; 2860 unsigned NumElems = Mask.getNumOperands(); 2861 for (unsigned i = 0; i != NumElems; ++i) { 2862 SDValue Arg = Mask.getOperand(i); 2863 if (Arg.getOpcode() != ISD::UNDEF) { 2864 unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); 2865 if (Val > NumElems) { 2866 Arg = DAG.getConstant(NumElems, Arg.getValueType()); 2867 Changed = true; 2868 } 2869 } 2870 MaskVec.push_back(Arg); 2871 } 2872 2873 if (Changed) 2874 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), 2875 &MaskVec[0], MaskVec.size()); 2876 return Mask; 2877} 2878 2879/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 2880/// operation of specified width. 2881static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { 2882 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2883 MVT BaseVT = MaskVT.getVectorElementType(); 2884 2885 SmallVector<SDValue, 8> MaskVec; 2886 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); 2887 for (unsigned i = 1; i != NumElems; ++i) 2888 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2889 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2890} 2891 2892/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation 2893/// of specified width. 2894static SDValue getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { 2895 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2896 MVT BaseVT = MaskVT.getVectorElementType(); 2897 SmallVector<SDValue, 8> MaskVec; 2898 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 2899 MaskVec.push_back(DAG.getConstant(i, BaseVT)); 2900 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); 2901 } 2902 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2903} 2904 2905/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation 2906/// of specified width. 2907static SDValue getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { 2908 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2909 MVT BaseVT = MaskVT.getVectorElementType(); 2910 unsigned Half = NumElems/2; 2911 SmallVector<SDValue, 8> MaskVec; 2912 for (unsigned i = 0; i != Half; ++i) { 2913 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); 2914 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); 2915 } 2916 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2917} 2918 2919/// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps 2920/// element #0 of a vector with the specified index, leaving the rest of the 2921/// elements in place. 2922static SDValue getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, 2923 SelectionDAG &DAG) { 2924 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2925 MVT BaseVT = MaskVT.getVectorElementType(); 2926 SmallVector<SDValue, 8> MaskVec; 2927 // Element #0 of the result gets the elt we are replacing. 2928 MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); 2929 for (unsigned i = 1; i != NumElems; ++i) 2930 MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); 2931 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); 2932} 2933 2934/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 2935static SDValue PromoteSplat(SDValue Op, SelectionDAG &DAG, bool HasSSE2) { 2936 MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; 2937 MVT VT = Op.getValueType(); 2938 if (PVT == VT) 2939 return Op; 2940 SDValue V1 = Op.getOperand(0); 2941 SDValue Mask = Op.getOperand(2); 2942 unsigned NumElems = Mask.getNumOperands(); 2943 // Special handling of v4f32 -> v4i32. 2944 if (VT != MVT::v4f32) { 2945 Mask = getUnpacklMask(NumElems, DAG); 2946 while (NumElems > 4) { 2947 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); 2948 NumElems >>= 1; 2949 } 2950 Mask = getZeroVector(MVT::v4i32, true, DAG); 2951 } 2952 2953 V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); 2954 SDValue Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, 2955 DAG.getNode(ISD::UNDEF, PVT), Mask); 2956 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); 2957} 2958 2959/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 2960/// vector of zero or undef vector. This produces a shuffle where the low 2961/// element of V2 is swizzled into the zero/undef vector, landing at element 2962/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 2963static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 2964 bool isZero, bool HasSSE2, 2965 SelectionDAG &DAG) { 2966 MVT VT = V2.getValueType(); 2967 SDValue V1 = isZero 2968 ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); 2969 unsigned NumElems = V2.getValueType().getVectorNumElements(); 2970 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 2971 MVT EVT = MaskVT.getVectorElementType(); 2972 SmallVector<SDValue, 16> MaskVec; 2973 for (unsigned i = 0; i != NumElems; ++i) 2974 if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. 2975 MaskVec.push_back(DAG.getConstant(NumElems, EVT)); 2976 else 2977 MaskVec.push_back(DAG.getConstant(i, EVT)); 2978 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 2979 &MaskVec[0], MaskVec.size()); 2980 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); 2981} 2982 2983/// getNumOfConsecutiveZeros - Return the number of elements in a result of 2984/// a shuffle that is zero. 2985static 2986unsigned getNumOfConsecutiveZeros(SDValue Op, SDValue Mask, 2987 unsigned NumElems, bool Low, 2988 SelectionDAG &DAG) { 2989 unsigned NumZeros = 0; 2990 for (unsigned i = 0; i < NumElems; ++i) { 2991 unsigned Index = Low ? i : NumElems-i-1; 2992 SDValue Idx = Mask.getOperand(Index); 2993 if (Idx.getOpcode() == ISD::UNDEF) { 2994 ++NumZeros; 2995 continue; 2996 } 2997 SDValue Elt = DAG.getShuffleScalarElt(Op.getNode(), Index); 2998 if (Elt.getNode() && isZeroNode(Elt)) 2999 ++NumZeros; 3000 else 3001 break; 3002 } 3003 return NumZeros; 3004} 3005 3006/// isVectorShift - Returns true if the shuffle can be implemented as a 3007/// logical left or right shift of a vector. 3008static bool isVectorShift(SDValue Op, SDValue Mask, SelectionDAG &DAG, 3009 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 3010 unsigned NumElems = Mask.getNumOperands(); 3011 3012 isLeft = true; 3013 unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); 3014 if (!NumZeros) { 3015 isLeft = false; 3016 NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); 3017 if (!NumZeros) 3018 return false; 3019 } 3020 3021 bool SeenV1 = false; 3022 bool SeenV2 = false; 3023 for (unsigned i = NumZeros; i < NumElems; ++i) { 3024 unsigned Val = isLeft ? (i - NumZeros) : i; 3025 SDValue Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); 3026 if (Idx.getOpcode() == ISD::UNDEF) 3027 continue; 3028 unsigned Index = cast<ConstantSDNode>(Idx)->getValue(); 3029 if (Index < NumElems) 3030 SeenV1 = true; 3031 else { 3032 Index -= NumElems; 3033 SeenV2 = true; 3034 } 3035 if (Index != Val) 3036 return false; 3037 } 3038 if (SeenV1 && SeenV2) 3039 return false; 3040 3041 ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); 3042 ShAmt = NumZeros; 3043 return true; 3044} 3045 3046 3047/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 3048/// 3049static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 3050 unsigned NumNonZero, unsigned NumZero, 3051 SelectionDAG &DAG, TargetLowering &TLI) { 3052 if (NumNonZero > 8) 3053 return SDValue(); 3054 3055 SDValue V(0, 0); 3056 bool First = true; 3057 for (unsigned i = 0; i < 16; ++i) { 3058 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 3059 if (ThisIsNonZero && First) { 3060 if (NumZero) 3061 V = getZeroVector(MVT::v8i16, true, DAG); 3062 else 3063 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3064 First = false; 3065 } 3066 3067 if ((i & 1) != 0) { 3068 SDValue ThisElt(0, 0), LastElt(0, 0); 3069 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 3070 if (LastIsNonZero) { 3071 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); 3072 } 3073 if (ThisIsNonZero) { 3074 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); 3075 ThisElt = DAG.getNode(ISD::SHL, MVT::i16, 3076 ThisElt, DAG.getConstant(8, MVT::i8)); 3077 if (LastIsNonZero) 3078 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); 3079 } else 3080 ThisElt = LastElt; 3081 3082 if (ThisElt.getNode()) 3083 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, 3084 DAG.getIntPtrConstant(i/2)); 3085 } 3086 } 3087 3088 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); 3089} 3090 3091/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 3092/// 3093static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 3094 unsigned NumNonZero, unsigned NumZero, 3095 SelectionDAG &DAG, TargetLowering &TLI) { 3096 if (NumNonZero > 4) 3097 return SDValue(); 3098 3099 SDValue V(0, 0); 3100 bool First = true; 3101 for (unsigned i = 0; i < 8; ++i) { 3102 bool isNonZero = (NonZeros & (1 << i)) != 0; 3103 if (isNonZero) { 3104 if (First) { 3105 if (NumZero) 3106 V = getZeroVector(MVT::v8i16, true, DAG); 3107 else 3108 V = DAG.getNode(ISD::UNDEF, MVT::v8i16); 3109 First = false; 3110 } 3111 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), 3112 DAG.getIntPtrConstant(i)); 3113 } 3114 } 3115 3116 return V; 3117} 3118 3119/// getVShift - Return a vector logical shift node. 3120/// 3121static SDValue getVShift(bool isLeft, MVT VT, SDValue SrcOp, 3122 unsigned NumBits, SelectionDAG &DAG, 3123 const TargetLowering &TLI) { 3124 bool isMMX = VT.getSizeInBits() == 64; 3125 MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 3126 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 3127 SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); 3128 return DAG.getNode(ISD::BIT_CONVERT, VT, 3129 DAG.getNode(Opc, ShVT, SrcOp, 3130 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 3131} 3132 3133SDValue 3134X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) { 3135 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 3136 if (ISD::isBuildVectorAllZeros(Op.getNode()) 3137 || ISD::isBuildVectorAllOnes(Op.getNode())) { 3138 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 3139 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 3140 // eliminated on x86-32 hosts. 3141 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 3142 return Op; 3143 3144 if (ISD::isBuildVectorAllOnes(Op.getNode())) 3145 return getOnesVector(Op.getValueType(), DAG); 3146 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); 3147 } 3148 3149 MVT VT = Op.getValueType(); 3150 MVT EVT = VT.getVectorElementType(); 3151 unsigned EVTBits = EVT.getSizeInBits(); 3152 3153 unsigned NumElems = Op.getNumOperands(); 3154 unsigned NumZero = 0; 3155 unsigned NumNonZero = 0; 3156 unsigned NonZeros = 0; 3157 bool IsAllConstants = true; 3158 SmallSet<SDValue, 8> Values; 3159 for (unsigned i = 0; i < NumElems; ++i) { 3160 SDValue Elt = Op.getOperand(i); 3161 if (Elt.getOpcode() == ISD::UNDEF) 3162 continue; 3163 Values.insert(Elt); 3164 if (Elt.getOpcode() != ISD::Constant && 3165 Elt.getOpcode() != ISD::ConstantFP) 3166 IsAllConstants = false; 3167 if (isZeroNode(Elt)) 3168 NumZero++; 3169 else { 3170 NonZeros |= (1 << i); 3171 NumNonZero++; 3172 } 3173 } 3174 3175 if (NumNonZero == 0) { 3176 // All undef vector. Return an UNDEF. All zero vectors were handled above. 3177 return DAG.getNode(ISD::UNDEF, VT); 3178 } 3179 3180 // Special case for single non-zero, non-undef, element. 3181 if (NumNonZero == 1 && NumElems <= 4) { 3182 unsigned Idx = CountTrailingZeros_32(NonZeros); 3183 SDValue Item = Op.getOperand(Idx); 3184 3185 // If this is an insertion of an i64 value on x86-32, and if the top bits of 3186 // the value are obviously zero, truncate the value to i32 and do the 3187 // insertion that way. Only do this if the value is non-constant or if the 3188 // value is a constant being inserted into element 0. It is cheaper to do 3189 // a constant pool load than it is to do a movd + shuffle. 3190 if (EVT == MVT::i64 && !Subtarget->is64Bit() && 3191 (!IsAllConstants || Idx == 0)) { 3192 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 3193 // Handle MMX and SSE both. 3194 MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 3195 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 3196 3197 // Truncate the value (which may itself be a constant) to i32, and 3198 // convert it to a vector with movd (S2V+shuffle to zero extend). 3199 Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); 3200 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); 3201 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 3202 Subtarget->hasSSE2(), DAG); 3203 3204 // Now we have our 32-bit value zero extended in the low element of 3205 // a vector. If Idx != 0, swizzle it into place. 3206 if (Idx != 0) { 3207 SDValue Ops[] = { 3208 Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), 3209 getSwapEltZeroMask(VecElts, Idx, DAG) 3210 }; 3211 Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); 3212 } 3213 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); 3214 } 3215 } 3216 3217 // If we have a constant or non-constant insertion into the low element of 3218 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 3219 // the rest of the elements. This will be matched as movd/movq/movss/movsd 3220 // depending on what the source datatype is. Because we can only get here 3221 // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. 3222 if (Idx == 0 && 3223 // Don't do this for i64 values on x86-32. 3224 (EVT != MVT::i64 || Subtarget->is64Bit())) { 3225 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3226 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 3227 return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3228 Subtarget->hasSSE2(), DAG); 3229 } 3230 3231 // Is it a vector logical left shift? 3232 if (NumElems == 2 && Idx == 1 && 3233 isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { 3234 unsigned NumBits = VT.getSizeInBits(); 3235 return getVShift(true, VT, 3236 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), 3237 NumBits/2, DAG, *this); 3238 } 3239 3240 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 3241 return SDValue(); 3242 3243 // Otherwise, if this is a vector with i32 or f32 elements, and the element 3244 // is a non-constant being inserted into an element other than the low one, 3245 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 3246 // movd/movss) to move this into the low element, then shuffle it into 3247 // place. 3248 if (EVTBits == 32) { 3249 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); 3250 3251 // Turn it into a shuffle of zero and zero-extended scalar to vector. 3252 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 3253 Subtarget->hasSSE2(), DAG); 3254 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3255 MVT MaskEVT = MaskVT.getVectorElementType(); 3256 SmallVector<SDValue, 8> MaskVec; 3257 for (unsigned i = 0; i < NumElems; i++) 3258 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); 3259 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3260 &MaskVec[0], MaskVec.size()); 3261 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, 3262 DAG.getNode(ISD::UNDEF, VT), Mask); 3263 } 3264 } 3265 3266 // Splat is obviously ok. Let legalizer expand it to a shuffle. 3267 if (Values.size() == 1) 3268 return SDValue(); 3269 3270 // A vector full of immediates; various special cases are already 3271 // handled, so this is best done with a single constant-pool load. 3272 if (IsAllConstants) 3273 return SDValue(); 3274 3275 // Let legalizer expand 2-wide build_vectors. 3276 if (EVTBits == 64) { 3277 if (NumNonZero == 1) { 3278 // One half is zero or undef. 3279 unsigned Idx = CountTrailingZeros_32(NonZeros); 3280 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, 3281 Op.getOperand(Idx)); 3282 return getShuffleVectorZeroOrUndef(V2, Idx, true, 3283 Subtarget->hasSSE2(), DAG); 3284 } 3285 return SDValue(); 3286 } 3287 3288 // If element VT is < 32 bits, convert it to inserts into a zero vector. 3289 if (EVTBits == 8 && NumElems == 16) { 3290 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 3291 *this); 3292 if (V.getNode()) return V; 3293 } 3294 3295 if (EVTBits == 16 && NumElems == 8) { 3296 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 3297 *this); 3298 if (V.getNode()) return V; 3299 } 3300 3301 // If element VT is == 32 bits, turn it into a number of shuffles. 3302 SmallVector<SDValue, 8> V; 3303 V.resize(NumElems); 3304 if (NumElems == 4 && NumZero > 0) { 3305 for (unsigned i = 0; i < 4; ++i) { 3306 bool isZero = !(NonZeros & (1 << i)); 3307 if (isZero) 3308 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3309 else 3310 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3311 } 3312 3313 for (unsigned i = 0; i < 2; ++i) { 3314 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 3315 default: break; 3316 case 0: 3317 V[i] = V[i*2]; // Must be a zero vector. 3318 break; 3319 case 1: 3320 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], 3321 getMOVLMask(NumElems, DAG)); 3322 break; 3323 case 2: 3324 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3325 getMOVLMask(NumElems, DAG)); 3326 break; 3327 case 3: 3328 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], 3329 getUnpacklMask(NumElems, DAG)); 3330 break; 3331 } 3332 } 3333 3334 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 3335 MVT EVT = MaskVT.getVectorElementType(); 3336 SmallVector<SDValue, 8> MaskVec; 3337 bool Reverse = (NonZeros & 0x3) == 2; 3338 for (unsigned i = 0; i < 2; ++i) 3339 if (Reverse) 3340 MaskVec.push_back(DAG.getConstant(1-i, EVT)); 3341 else 3342 MaskVec.push_back(DAG.getConstant(i, EVT)); 3343 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 3344 for (unsigned i = 0; i < 2; ++i) 3345 if (Reverse) 3346 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); 3347 else 3348 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); 3349 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3350 &MaskVec[0], MaskVec.size()); 3351 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); 3352 } 3353 3354 if (Values.size() > 2) { 3355 // Expand into a number of unpckl*. 3356 // e.g. for v4f32 3357 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 3358 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 3359 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 3360 SDValue UnpckMask = getUnpacklMask(NumElems, DAG); 3361 for (unsigned i = 0; i < NumElems; ++i) 3362 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); 3363 NumElems >>= 1; 3364 while (NumElems != 0) { 3365 for (unsigned i = 0; i < NumElems; ++i) 3366 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], 3367 UnpckMask); 3368 NumElems >>= 1; 3369 } 3370 return V[0]; 3371 } 3372 3373 return SDValue(); 3374} 3375 3376static 3377SDValue LowerVECTOR_SHUFFLEv8i16(SDValue V1, SDValue V2, 3378 SDValue PermMask, SelectionDAG &DAG, 3379 TargetLowering &TLI) { 3380 SDValue NewV; 3381 MVT MaskVT = MVT::getIntVectorWithNumElements(8); 3382 MVT MaskEVT = MaskVT.getVectorElementType(); 3383 MVT PtrVT = TLI.getPointerTy(); 3384 SmallVector<SDValue, 8> MaskElts(PermMask.getNode()->op_begin(), 3385 PermMask.getNode()->op_end()); 3386 3387 // First record which half of which vector the low elements come from. 3388 SmallVector<unsigned, 4> LowQuad(4); 3389 for (unsigned i = 0; i < 4; ++i) { 3390 SDValue Elt = MaskElts[i]; 3391 if (Elt.getOpcode() == ISD::UNDEF) 3392 continue; 3393 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3394 int QuadIdx = EltIdx / 4; 3395 ++LowQuad[QuadIdx]; 3396 } 3397 3398 int BestLowQuad = -1; 3399 unsigned MaxQuad = 1; 3400 for (unsigned i = 0; i < 4; ++i) { 3401 if (LowQuad[i] > MaxQuad) { 3402 BestLowQuad = i; 3403 MaxQuad = LowQuad[i]; 3404 } 3405 } 3406 3407 // Record which half of which vector the high elements come from. 3408 SmallVector<unsigned, 4> HighQuad(4); 3409 for (unsigned i = 4; i < 8; ++i) { 3410 SDValue Elt = MaskElts[i]; 3411 if (Elt.getOpcode() == ISD::UNDEF) 3412 continue; 3413 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3414 int QuadIdx = EltIdx / 4; 3415 ++HighQuad[QuadIdx]; 3416 } 3417 3418 int BestHighQuad = -1; 3419 MaxQuad = 1; 3420 for (unsigned i = 0; i < 4; ++i) { 3421 if (HighQuad[i] > MaxQuad) { 3422 BestHighQuad = i; 3423 MaxQuad = HighQuad[i]; 3424 } 3425 } 3426 3427 // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. 3428 if (BestLowQuad != -1 || BestHighQuad != -1) { 3429 // First sort the 4 chunks in order using shufpd. 3430 SmallVector<SDValue, 8> MaskVec; 3431 3432 if (BestLowQuad != -1) 3433 MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); 3434 else 3435 MaskVec.push_back(DAG.getConstant(0, MVT::i32)); 3436 3437 if (BestHighQuad != -1) 3438 MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); 3439 else 3440 MaskVec.push_back(DAG.getConstant(1, MVT::i32)); 3441 3442 SDValue Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); 3443 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, 3444 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), 3445 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); 3446 NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); 3447 3448 // Now sort high and low parts separately. 3449 BitVector InOrder(8); 3450 if (BestLowQuad != -1) { 3451 // Sort lower half in order using PSHUFLW. 3452 MaskVec.clear(); 3453 bool AnyOutOrder = false; 3454 3455 for (unsigned i = 0; i != 4; ++i) { 3456 SDValue Elt = MaskElts[i]; 3457 if (Elt.getOpcode() == ISD::UNDEF) { 3458 MaskVec.push_back(Elt); 3459 InOrder.set(i); 3460 } else { 3461 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3462 if (EltIdx != i) 3463 AnyOutOrder = true; 3464 3465 MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); 3466 3467 // If this element is in the right place after this shuffle, then 3468 // remember it. 3469 if ((int)(EltIdx / 4) == BestLowQuad) 3470 InOrder.set(i); 3471 } 3472 } 3473 if (AnyOutOrder) { 3474 for (unsigned i = 4; i != 8; ++i) 3475 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3476 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3477 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3478 } 3479 } 3480 3481 if (BestHighQuad != -1) { 3482 // Sort high half in order using PSHUFHW if possible. 3483 MaskVec.clear(); 3484 3485 for (unsigned i = 0; i != 4; ++i) 3486 MaskVec.push_back(DAG.getConstant(i, MaskEVT)); 3487 3488 bool AnyOutOrder = false; 3489 for (unsigned i = 4; i != 8; ++i) { 3490 SDValue Elt = MaskElts[i]; 3491 if (Elt.getOpcode() == ISD::UNDEF) { 3492 MaskVec.push_back(Elt); 3493 InOrder.set(i); 3494 } else { 3495 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3496 if (EltIdx != i) 3497 AnyOutOrder = true; 3498 3499 MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); 3500 3501 // If this element is in the right place after this shuffle, then 3502 // remember it. 3503 if ((int)(EltIdx / 4) == BestHighQuad) 3504 InOrder.set(i); 3505 } 3506 } 3507 3508 if (AnyOutOrder) { 3509 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3510 NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); 3511 } 3512 } 3513 3514 // The other elements are put in the right place using pextrw and pinsrw. 3515 for (unsigned i = 0; i != 8; ++i) { 3516 if (InOrder[i]) 3517 continue; 3518 SDValue Elt = MaskElts[i]; 3519 if (Elt.getOpcode() == ISD::UNDEF) 3520 continue; 3521 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3522 SDValue ExtOp = (EltIdx < 8) 3523 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3524 DAG.getConstant(EltIdx, PtrVT)) 3525 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3526 DAG.getConstant(EltIdx - 8, PtrVT)); 3527 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3528 DAG.getConstant(i, PtrVT)); 3529 } 3530 3531 return NewV; 3532 } 3533 3534 // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use as 3535 // few as possible. First, let's find out how many elements are already in the 3536 // right order. 3537 unsigned V1InOrder = 0; 3538 unsigned V1FromV1 = 0; 3539 unsigned V2InOrder = 0; 3540 unsigned V2FromV2 = 0; 3541 SmallVector<SDValue, 8> V1Elts; 3542 SmallVector<SDValue, 8> V2Elts; 3543 for (unsigned i = 0; i < 8; ++i) { 3544 SDValue Elt = MaskElts[i]; 3545 if (Elt.getOpcode() == ISD::UNDEF) { 3546 V1Elts.push_back(Elt); 3547 V2Elts.push_back(Elt); 3548 ++V1InOrder; 3549 ++V2InOrder; 3550 continue; 3551 } 3552 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3553 if (EltIdx == i) { 3554 V1Elts.push_back(Elt); 3555 V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); 3556 ++V1InOrder; 3557 } else if (EltIdx == i+8) { 3558 V1Elts.push_back(Elt); 3559 V2Elts.push_back(DAG.getConstant(i, MaskEVT)); 3560 ++V2InOrder; 3561 } else if (EltIdx < 8) { 3562 V1Elts.push_back(Elt); 3563 ++V1FromV1; 3564 } else { 3565 V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); 3566 ++V2FromV2; 3567 } 3568 } 3569 3570 if (V2InOrder > V1InOrder) { 3571 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3572 std::swap(V1, V2); 3573 std::swap(V1Elts, V2Elts); 3574 std::swap(V1FromV1, V2FromV2); 3575 } 3576 3577 if ((V1FromV1 + V1InOrder) != 8) { 3578 // Some elements are from V2. 3579 if (V1FromV1) { 3580 // If there are elements that are from V1 but out of place, 3581 // then first sort them in place 3582 SmallVector<SDValue, 8> MaskVec; 3583 for (unsigned i = 0; i < 8; ++i) { 3584 SDValue Elt = V1Elts[i]; 3585 if (Elt.getOpcode() == ISD::UNDEF) { 3586 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3587 continue; 3588 } 3589 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3590 if (EltIdx >= 8) 3591 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3592 else 3593 MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); 3594 } 3595 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); 3596 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); 3597 } 3598 3599 NewV = V1; 3600 for (unsigned i = 0; i < 8; ++i) { 3601 SDValue Elt = V1Elts[i]; 3602 if (Elt.getOpcode() == ISD::UNDEF) 3603 continue; 3604 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3605 if (EltIdx < 8) 3606 continue; 3607 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, 3608 DAG.getConstant(EltIdx - 8, PtrVT)); 3609 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3610 DAG.getConstant(i, PtrVT)); 3611 } 3612 return NewV; 3613 } else { 3614 // All elements are from V1. 3615 NewV = V1; 3616 for (unsigned i = 0; i < 8; ++i) { 3617 SDValue Elt = V1Elts[i]; 3618 if (Elt.getOpcode() == ISD::UNDEF) 3619 continue; 3620 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3621 SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, 3622 DAG.getConstant(EltIdx, PtrVT)); 3623 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, 3624 DAG.getConstant(i, PtrVT)); 3625 } 3626 return NewV; 3627 } 3628} 3629 3630/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 3631/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 3632/// done when every pair / quad of shuffle mask elements point to elements in 3633/// the right sequence. e.g. 3634/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 3635static 3636SDValue RewriteAsNarrowerShuffle(SDValue V1, SDValue V2, 3637 MVT VT, 3638 SDValue PermMask, SelectionDAG &DAG, 3639 TargetLowering &TLI) { 3640 unsigned NumElems = PermMask.getNumOperands(); 3641 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 3642 MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 3643 MVT MaskEltVT = MaskVT.getVectorElementType(); 3644 MVT NewVT = MaskVT; 3645 switch (VT.getSimpleVT()) { 3646 default: assert(false && "Unexpected!"); 3647 case MVT::v4f32: NewVT = MVT::v2f64; break; 3648 case MVT::v4i32: NewVT = MVT::v2i64; break; 3649 case MVT::v8i16: NewVT = MVT::v4i32; break; 3650 case MVT::v16i8: NewVT = MVT::v4i32; break; 3651 } 3652 3653 if (NewWidth == 2) { 3654 if (VT.isInteger()) 3655 NewVT = MVT::v2i64; 3656 else 3657 NewVT = MVT::v2f64; 3658 } 3659 unsigned Scale = NumElems / NewWidth; 3660 SmallVector<SDValue, 8> MaskVec; 3661 for (unsigned i = 0; i < NumElems; i += Scale) { 3662 unsigned StartIdx = ~0U; 3663 for (unsigned j = 0; j < Scale; ++j) { 3664 SDValue Elt = PermMask.getOperand(i+j); 3665 if (Elt.getOpcode() == ISD::UNDEF) 3666 continue; 3667 unsigned EltIdx = cast<ConstantSDNode>(Elt)->getValue(); 3668 if (StartIdx == ~0U) 3669 StartIdx = EltIdx - (EltIdx % Scale); 3670 if (EltIdx != StartIdx + j) 3671 return SDValue(); 3672 } 3673 if (StartIdx == ~0U) 3674 MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEltVT)); 3675 else 3676 MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MaskEltVT)); 3677 } 3678 3679 V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); 3680 V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); 3681 return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, 3682 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3683 &MaskVec[0], MaskVec.size())); 3684} 3685 3686/// getVZextMovL - Return a zero-extending vector move low node. 3687/// 3688static SDValue getVZextMovL(MVT VT, MVT OpVT, 3689 SDValue SrcOp, SelectionDAG &DAG, 3690 const X86Subtarget *Subtarget) { 3691 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 3692 LoadSDNode *LD = NULL; 3693 if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) 3694 LD = dyn_cast<LoadSDNode>(SrcOp); 3695 if (!LD) { 3696 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 3697 // instead. 3698 MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 3699 if ((EVT != MVT::i64 || Subtarget->is64Bit()) && 3700 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 3701 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 3702 SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { 3703 // PR2108 3704 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 3705 return DAG.getNode(ISD::BIT_CONVERT, VT, 3706 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3707 DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, 3708 SrcOp.getOperand(0) 3709 .getOperand(0)))); 3710 } 3711 } 3712 } 3713 3714 return DAG.getNode(ISD::BIT_CONVERT, VT, 3715 DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, 3716 DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); 3717} 3718 3719/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of 3720/// shuffles. 3721static SDValue 3722LowerVECTOR_SHUFFLE_4wide(SDValue V1, SDValue V2, 3723 SDValue PermMask, MVT VT, SelectionDAG &DAG) { 3724 MVT MaskVT = PermMask.getValueType(); 3725 MVT MaskEVT = MaskVT.getVectorElementType(); 3726 SmallVector<std::pair<int, int>, 8> Locs; 3727 Locs.resize(4); 3728 SmallVector<SDValue, 8> Mask1(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3729 unsigned NumHi = 0; 3730 unsigned NumLo = 0; 3731 for (unsigned i = 0; i != 4; ++i) { 3732 SDValue Elt = PermMask.getOperand(i); 3733 if (Elt.getOpcode() == ISD::UNDEF) { 3734 Locs[i] = std::make_pair(-1, -1); 3735 } else { 3736 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3737 assert(Val < 8 && "Invalid VECTOR_SHUFFLE index!"); 3738 if (Val < 4) { 3739 Locs[i] = std::make_pair(0, NumLo); 3740 Mask1[NumLo] = Elt; 3741 NumLo++; 3742 } else { 3743 Locs[i] = std::make_pair(1, NumHi); 3744 if (2+NumHi < 4) 3745 Mask1[2+NumHi] = Elt; 3746 NumHi++; 3747 } 3748 } 3749 } 3750 3751 if (NumLo <= 2 && NumHi <= 2) { 3752 // If no more than two elements come from either vector. This can be 3753 // implemented with two shuffles. First shuffle gather the elements. 3754 // The second shuffle, which takes the first shuffle as both of its 3755 // vector operands, put the elements into the right order. 3756 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3757 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3758 &Mask1[0], Mask1.size())); 3759 3760 SmallVector<SDValue, 8> Mask2(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3761 for (unsigned i = 0; i != 4; ++i) { 3762 if (Locs[i].first == -1) 3763 continue; 3764 else { 3765 unsigned Idx = (i < 2) ? 0 : 4; 3766 Idx += Locs[i].first * 2 + Locs[i].second; 3767 Mask2[i] = DAG.getConstant(Idx, MaskEVT); 3768 } 3769 } 3770 3771 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, 3772 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3773 &Mask2[0], Mask2.size())); 3774 } else if (NumLo == 3 || NumHi == 3) { 3775 // Otherwise, we must have three elements from one vector, call it X, and 3776 // one element from the other, call it Y. First, use a shufps to build an 3777 // intermediate vector with the one element from Y and the element from X 3778 // that will be in the same half in the final destination (the indexes don't 3779 // matter). Then, use a shufps to build the final vector, taking the half 3780 // containing the element from Y from the intermediate, and the other half 3781 // from X. 3782 if (NumHi == 3) { 3783 // Normalize it so the 3 elements come from V1. 3784 PermMask = CommuteVectorShuffleMask(PermMask, DAG); 3785 std::swap(V1, V2); 3786 } 3787 3788 // Find the element from V2. 3789 unsigned HiIndex; 3790 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 3791 SDValue Elt = PermMask.getOperand(HiIndex); 3792 if (Elt.getOpcode() == ISD::UNDEF) 3793 continue; 3794 unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); 3795 if (Val >= 4) 3796 break; 3797 } 3798 3799 Mask1[0] = PermMask.getOperand(HiIndex); 3800 Mask1[1] = DAG.getNode(ISD::UNDEF, MaskEVT); 3801 Mask1[2] = PermMask.getOperand(HiIndex^1); 3802 Mask1[3] = DAG.getNode(ISD::UNDEF, MaskEVT); 3803 V2 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3804 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3805 3806 if (HiIndex >= 2) { 3807 Mask1[0] = PermMask.getOperand(0); 3808 Mask1[1] = PermMask.getOperand(1); 3809 Mask1[2] = DAG.getConstant(HiIndex & 1 ? 6 : 4, MaskEVT); 3810 Mask1[3] = DAG.getConstant(HiIndex & 1 ? 4 : 6, MaskEVT); 3811 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3812 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3813 } else { 3814 Mask1[0] = DAG.getConstant(HiIndex & 1 ? 2 : 0, MaskEVT); 3815 Mask1[1] = DAG.getConstant(HiIndex & 1 ? 0 : 2, MaskEVT); 3816 Mask1[2] = PermMask.getOperand(2); 3817 Mask1[3] = PermMask.getOperand(3); 3818 if (Mask1[2].getOpcode() != ISD::UNDEF) 3819 Mask1[2] = DAG.getConstant(cast<ConstantSDNode>(Mask1[2])->getValue()+4, 3820 MaskEVT); 3821 if (Mask1[3].getOpcode() != ISD::UNDEF) 3822 Mask1[3] = DAG.getConstant(cast<ConstantSDNode>(Mask1[3])->getValue()+4, 3823 MaskEVT); 3824 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, 3825 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); 3826 } 3827 } 3828 3829 // Break it into (shuffle shuffle_hi, shuffle_lo). 3830 Locs.clear(); 3831 SmallVector<SDValue,8> LoMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3832 SmallVector<SDValue,8> HiMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); 3833 SmallVector<SDValue,8> *MaskPtr = &LoMask; 3834 unsigned MaskIdx = 0; 3835 unsigned LoIdx = 0; 3836 unsigned HiIdx = 2; 3837 for (unsigned i = 0; i != 4; ++i) { 3838 if (i == 2) { 3839 MaskPtr = &HiMask; 3840 MaskIdx = 1; 3841 LoIdx = 0; 3842 HiIdx = 2; 3843 } 3844 SDValue Elt = PermMask.getOperand(i); 3845 if (Elt.getOpcode() == ISD::UNDEF) { 3846 Locs[i] = std::make_pair(-1, -1); 3847 } else if (cast<ConstantSDNode>(Elt)->getValue() < 4) { 3848 Locs[i] = std::make_pair(MaskIdx, LoIdx); 3849 (*MaskPtr)[LoIdx] = Elt; 3850 LoIdx++; 3851 } else { 3852 Locs[i] = std::make_pair(MaskIdx, HiIdx); 3853 (*MaskPtr)[HiIdx] = Elt; 3854 HiIdx++; 3855 } 3856 } 3857 3858 SDValue LoShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3859 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3860 &LoMask[0], LoMask.size())); 3861 SDValue HiShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, 3862 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3863 &HiMask[0], HiMask.size())); 3864 SmallVector<SDValue, 8> MaskOps; 3865 for (unsigned i = 0; i != 4; ++i) { 3866 if (Locs[i].first == -1) { 3867 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); 3868 } else { 3869 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 3870 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); 3871 } 3872 } 3873 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, 3874 DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 3875 &MaskOps[0], MaskOps.size())); 3876} 3877 3878SDValue 3879X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 3880 SDValue V1 = Op.getOperand(0); 3881 SDValue V2 = Op.getOperand(1); 3882 SDValue PermMask = Op.getOperand(2); 3883 MVT VT = Op.getValueType(); 3884 unsigned NumElems = PermMask.getNumOperands(); 3885 bool isMMX = VT.getSizeInBits() == 64; 3886 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 3887 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 3888 bool V1IsSplat = false; 3889 bool V2IsSplat = false; 3890 3891 if (isUndefShuffle(Op.getNode())) 3892 return DAG.getNode(ISD::UNDEF, VT); 3893 3894 if (isZeroShuffle(Op.getNode())) 3895 return getZeroVector(VT, Subtarget->hasSSE2(), DAG); 3896 3897 if (isIdentityMask(PermMask.getNode())) 3898 return V1; 3899 else if (isIdentityMask(PermMask.getNode(), true)) 3900 return V2; 3901 3902 if (isSplatMask(PermMask.getNode())) { 3903 if (isMMX || NumElems < 4) return Op; 3904 // Promote it to a v4{if}32 splat. 3905 return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); 3906 } 3907 3908 // If the shuffle can be profitably rewritten as a narrower shuffle, then 3909 // do it! 3910 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 3911 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); 3912 if (NewOp.getNode()) 3913 return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); 3914 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 3915 // FIXME: Figure out a cleaner way to do this. 3916 // Try to make use of movq to zero out the top part. 3917 if (ISD::isBuildVectorAllZeros(V2.getNode())) { 3918 SDValue NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3919 DAG, *this); 3920 if (NewOp.getNode()) { 3921 SDValue NewV1 = NewOp.getOperand(0); 3922 SDValue NewV2 = NewOp.getOperand(1); 3923 SDValue NewMask = NewOp.getOperand(2); 3924 if (isCommutedMOVL(NewMask.getNode(), true, false)) { 3925 NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); 3926 return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); 3927 } 3928 } 3929 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { 3930 SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, 3931 DAG, *this); 3932 if (NewOp.getNode() && X86::isMOVLMask(NewOp.getOperand(2).getNode())) 3933 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 3934 DAG, Subtarget); 3935 } 3936 } 3937 3938 // Check if this can be converted into a logical shift. 3939 bool isLeft = false; 3940 unsigned ShAmt = 0; 3941 SDValue ShVal; 3942 bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); 3943 if (isShift && ShVal.hasOneUse()) { 3944 // If the shifted value has multiple uses, it may be cheaper to use 3945 // v_set0 + movlhps or movhlps, etc. 3946 MVT EVT = VT.getVectorElementType(); 3947 ShAmt *= EVT.getSizeInBits(); 3948 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3949 } 3950 3951 if (X86::isMOVLMask(PermMask.getNode())) { 3952 if (V1IsUndef) 3953 return V2; 3954 if (ISD::isBuildVectorAllZeros(V1.getNode())) 3955 return getVZextMovL(VT, VT, V2, DAG, Subtarget); 3956 if (!isMMX) 3957 return Op; 3958 } 3959 3960 if (!isMMX && (X86::isMOVSHDUPMask(PermMask.getNode()) || 3961 X86::isMOVSLDUPMask(PermMask.getNode()) || 3962 X86::isMOVHLPSMask(PermMask.getNode()) || 3963 X86::isMOVHPMask(PermMask.getNode()) || 3964 X86::isMOVLPMask(PermMask.getNode()))) 3965 return Op; 3966 3967 if (ShouldXformToMOVHLPS(PermMask.getNode()) || 3968 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), PermMask.getNode())) 3969 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3970 3971 if (isShift) { 3972 // No better options. Use a vshl / vsrl. 3973 MVT EVT = VT.getVectorElementType(); 3974 ShAmt *= EVT.getSizeInBits(); 3975 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); 3976 } 3977 3978 bool Commuted = false; 3979 // FIXME: This should also accept a bitcast of a splat? Be careful, not 3980 // 1,1,1,1 -> v8i16 though. 3981 V1IsSplat = isSplatVector(V1.getNode()); 3982 V2IsSplat = isSplatVector(V2.getNode()); 3983 3984 // Canonicalize the splat or undef, if present, to be on the RHS. 3985 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 3986 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3987 std::swap(V1IsSplat, V2IsSplat); 3988 std::swap(V1IsUndef, V2IsUndef); 3989 Commuted = true; 3990 } 3991 3992 // FIXME: Figure out a cleaner way to do this. 3993 if (isCommutedMOVL(PermMask.getNode(), V2IsSplat, V2IsUndef)) { 3994 if (V2IsUndef) return V1; 3995 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 3996 if (V2IsSplat) { 3997 // V2 is a splat, so the mask may be malformed. That is, it may point 3998 // to any V2 element. The instruction selectior won't like this. Get 3999 // a corrected mask and commute to form a proper MOVS{S|D}. 4000 SDValue NewMask = getMOVLMask(NumElems, DAG); 4001 if (NewMask.getNode() != PermMask.getNode()) 4002 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4003 } 4004 return Op; 4005 } 4006 4007 if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || 4008 X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || 4009 X86::isUNPCKLMask(PermMask.getNode()) || 4010 X86::isUNPCKHMask(PermMask.getNode())) 4011 return Op; 4012 4013 if (V2IsSplat) { 4014 // Normalize mask so all entries that point to V2 points to its first 4015 // element then try to match unpck{h|l} again. If match, return a 4016 // new vector_shuffle with the corrected mask. 4017 SDValue NewMask = NormalizeMask(PermMask, DAG); 4018 if (NewMask.getNode() != PermMask.getNode()) { 4019 if (X86::isUNPCKLMask(PermMask.getNode(), true)) { 4020 SDValue NewMask = getUnpacklMask(NumElems, DAG); 4021 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4022 } else if (X86::isUNPCKHMask(PermMask.getNode(), true)) { 4023 SDValue NewMask = getUnpackhMask(NumElems, DAG); 4024 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); 4025 } 4026 } 4027 } 4028 4029 // Normalize the node to match x86 shuffle ops if needed 4030 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.getNode())) 4031 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 4032 4033 if (Commuted) { 4034 // Commute is back and try unpck* again. 4035 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); 4036 if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || 4037 X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || 4038 X86::isUNPCKLMask(PermMask.getNode()) || 4039 X86::isUNPCKHMask(PermMask.getNode())) 4040 return Op; 4041 } 4042 4043 // Try PSHUF* first, then SHUFP*. 4044 // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically 4045 // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. 4046 if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.getNode())) { 4047 if (V2.getOpcode() != ISD::UNDEF) 4048 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, 4049 DAG.getNode(ISD::UNDEF, VT), PermMask); 4050 return Op; 4051 } 4052 4053 if (!isMMX) { 4054 if (Subtarget->hasSSE2() && 4055 (X86::isPSHUFDMask(PermMask.getNode()) || 4056 X86::isPSHUFHWMask(PermMask.getNode()) || 4057 X86::isPSHUFLWMask(PermMask.getNode()))) { 4058 MVT RVT = VT; 4059 if (VT == MVT::v4f32) { 4060 RVT = MVT::v4i32; 4061 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, 4062 DAG.getNode(ISD::BIT_CONVERT, RVT, V1), 4063 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4064 } else if (V2.getOpcode() != ISD::UNDEF) 4065 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, 4066 DAG.getNode(ISD::UNDEF, RVT), PermMask); 4067 if (RVT != VT) 4068 Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); 4069 return Op; 4070 } 4071 4072 // Binary or unary shufps. 4073 if (X86::isSHUFPMask(PermMask.getNode()) || 4074 (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.getNode()))) 4075 return Op; 4076 } 4077 4078 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 4079 if (VT == MVT::v8i16) { 4080 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); 4081 if (NewOp.getNode()) 4082 return NewOp; 4083 } 4084 4085 // Handle all 4 wide cases with a number of shuffles except for MMX. 4086 if (NumElems == 4 && !isMMX) 4087 return LowerVECTOR_SHUFFLE_4wide(V1, V2, PermMask, VT, DAG); 4088 4089 return SDValue(); 4090} 4091 4092SDValue 4093X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 4094 SelectionDAG &DAG) { 4095 MVT VT = Op.getValueType(); 4096 if (VT.getSizeInBits() == 8) { 4097 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, 4098 Op.getOperand(0), Op.getOperand(1)); 4099 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4100 DAG.getValueType(VT)); 4101 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4102 } else if (VT.getSizeInBits() == 16) { 4103 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, 4104 Op.getOperand(0), Op.getOperand(1)); 4105 SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, 4106 DAG.getValueType(VT)); 4107 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4108 } else if (VT == MVT::f32) { 4109 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 4110 // the result back to FR32 register. It's only worth matching if the 4111 // result has a single use which is a store or a bitcast to i32. 4112 if (!Op.hasOneUse()) 4113 return SDValue(); 4114 SDNode *User = *Op.getNode()->use_begin(); 4115 if (User->getOpcode() != ISD::STORE && 4116 (User->getOpcode() != ISD::BIT_CONVERT || 4117 User->getValueType(0) != MVT::i32)) 4118 return SDValue(); 4119 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4120 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), 4121 Op.getOperand(1)); 4122 return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); 4123 } 4124 return SDValue(); 4125} 4126 4127 4128SDValue 4129X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4130 if (!isa<ConstantSDNode>(Op.getOperand(1))) 4131 return SDValue(); 4132 4133 if (Subtarget->hasSSE41()) { 4134 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 4135 if (Res.getNode()) 4136 return Res; 4137 } 4138 4139 MVT VT = Op.getValueType(); 4140 // TODO: handle v16i8. 4141 if (VT.getSizeInBits() == 16) { 4142 SDValue Vec = Op.getOperand(0); 4143 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4144 if (Idx == 0) 4145 return DAG.getNode(ISD::TRUNCATE, MVT::i16, 4146 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4147 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), 4148 Op.getOperand(1))); 4149 // Transform it so it match pextrw which produces a 32-bit result. 4150 MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); 4151 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, EVT, 4152 Op.getOperand(0), Op.getOperand(1)); 4153 SDValue Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, 4154 DAG.getValueType(VT)); 4155 return DAG.getNode(ISD::TRUNCATE, VT, Assert); 4156 } else if (VT.getSizeInBits() == 32) { 4157 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4158 if (Idx == 0) 4159 return Op; 4160 // SHUFPS the element to the lowest double word, then movss. 4161 MVT MaskVT = MVT::getIntVectorWithNumElements(4); 4162 SmallVector<SDValue, 8> IdxVec; 4163 IdxVec. 4164 push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); 4165 IdxVec. 4166 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4167 IdxVec. 4168 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4169 IdxVec. 4170 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4171 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4172 &IdxVec[0], IdxVec.size()); 4173 SDValue Vec = Op.getOperand(0); 4174 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4175 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4176 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4177 DAG.getIntPtrConstant(0)); 4178 } else if (VT.getSizeInBits() == 64) { 4179 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 4180 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 4181 // to match extract_elt for f64. 4182 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); 4183 if (Idx == 0) 4184 return Op; 4185 4186 // UNPCKHPD the element to the lowest double word, then movsd. 4187 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 4188 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 4189 MVT MaskVT = MVT::getIntVectorWithNumElements(2); 4190 SmallVector<SDValue, 8> IdxVec; 4191 IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); 4192 IdxVec. 4193 push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); 4194 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 4195 &IdxVec[0], IdxVec.size()); 4196 SDValue Vec = Op.getOperand(0); 4197 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), 4198 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); 4199 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, 4200 DAG.getIntPtrConstant(0)); 4201 } 4202 4203 return SDValue(); 4204} 4205 4206SDValue 4207X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){ 4208 MVT VT = Op.getValueType(); 4209 MVT EVT = VT.getVectorElementType(); 4210 4211 SDValue N0 = Op.getOperand(0); 4212 SDValue N1 = Op.getOperand(1); 4213 SDValue N2 = Op.getOperand(2); 4214 4215 if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) && 4216 isa<ConstantSDNode>(N2)) { 4217 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 4218 : X86ISD::PINSRW; 4219 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 4220 // argument. 4221 if (N1.getValueType() != MVT::i32) 4222 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4223 if (N2.getValueType() != MVT::i32) 4224 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4225 return DAG.getNode(Opc, VT, N0, N1, N2); 4226 } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 4227 // Bits [7:6] of the constant are the source select. This will always be 4228 // zero here. The DAG Combiner may combine an extract_elt index into these 4229 // bits. For example (insert (extract, 3), 2) could be matched by putting 4230 // the '3' into bits [7:6] of X86ISD::INSERTPS. 4231 // Bits [5:4] of the constant are the destination select. This is the 4232 // value of the incoming immediate. 4233 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 4234 // combine either bitwise AND or insert of float 0.0 to set these bits. 4235 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue() << 4); 4236 return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); 4237 } 4238 return SDValue(); 4239} 4240 4241SDValue 4242X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 4243 MVT VT = Op.getValueType(); 4244 MVT EVT = VT.getVectorElementType(); 4245 4246 if (Subtarget->hasSSE41()) 4247 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 4248 4249 if (EVT == MVT::i8) 4250 return SDValue(); 4251 4252 SDValue N0 = Op.getOperand(0); 4253 SDValue N1 = Op.getOperand(1); 4254 SDValue N2 = Op.getOperand(2); 4255 4256 if (EVT.getSizeInBits() == 16) { 4257 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 4258 // as its second argument. 4259 if (N1.getValueType() != MVT::i32) 4260 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); 4261 if (N2.getValueType() != MVT::i32) 4262 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getValue()); 4263 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); 4264 } 4265 return SDValue(); 4266} 4267 4268SDValue 4269X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) { 4270 if (Op.getValueType() == MVT::v2f32) 4271 return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f32, 4272 DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i32, 4273 DAG.getNode(ISD::BIT_CONVERT, MVT::i32, 4274 Op.getOperand(0)))); 4275 4276 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); 4277 MVT VT = MVT::v2i32; 4278 switch (Op.getValueType().getSimpleVT()) { 4279 default: break; 4280 case MVT::v16i8: 4281 case MVT::v8i16: 4282 VT = MVT::v4i32; 4283 break; 4284 } 4285 return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), 4286 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); 4287} 4288 4289// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 4290// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 4291// one of the above mentioned nodes. It has to be wrapped because otherwise 4292// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 4293// be used to form addressing mode. These wrapped nodes will be selected 4294// into MOV32ri. 4295SDValue 4296X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) { 4297 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4298 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), 4299 getPointerTy(), 4300 CP->getAlignment()); 4301 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4302 // With PIC, the address is actually $g + Offset. 4303 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4304 !Subtarget->isPICStyleRIPRel()) { 4305 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4306 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4307 Result); 4308 } 4309 4310 return Result; 4311} 4312 4313SDValue 4314X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) { 4315 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 4316 SDValue Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); 4317 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4318 // With PIC, the address is actually $g + Offset. 4319 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4320 !Subtarget->isPICStyleRIPRel()) { 4321 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4322 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4323 Result); 4324 } 4325 4326 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to 4327 // load the value at address GV, not the value of GV itself. This means that 4328 // the GlobalAddress must be in the base or index register of the address, not 4329 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call 4330 // The same applies for external symbols during PIC codegen 4331 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false)) 4332 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, 4333 PseudoSourceValue::getGOT(), 0); 4334 4335 return Result; 4336} 4337 4338// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 4339static SDValue 4340LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4341 const MVT PtrVT) { 4342 SDValue InFlag; 4343 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, 4344 DAG.getNode(X86ISD::GlobalBaseReg, 4345 PtrVT), InFlag); 4346 InFlag = Chain.getValue(1); 4347 4348 // emit leal symbol@TLSGD(,%ebx,1), %eax 4349 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4350 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4351 GA->getValueType(0), 4352 GA->getOffset()); 4353 SDValue Ops[] = { Chain, TGA, InFlag }; 4354 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); 4355 InFlag = Result.getValue(2); 4356 Chain = Result.getValue(1); 4357 4358 // call ___tls_get_addr. This function receives its argument in 4359 // the register EAX. 4360 Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); 4361 InFlag = Chain.getValue(1); 4362 4363 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4364 SDValue Ops1[] = { Chain, 4365 DAG.getTargetExternalSymbol("___tls_get_addr", 4366 PtrVT), 4367 DAG.getRegister(X86::EAX, PtrVT), 4368 DAG.getRegister(X86::EBX, PtrVT), 4369 InFlag }; 4370 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); 4371 InFlag = Chain.getValue(1); 4372 4373 return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); 4374} 4375 4376// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 4377static SDValue 4378LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4379 const MVT PtrVT) { 4380 SDValue InFlag, Chain; 4381 4382 // emit leaq symbol@TLSGD(%rip), %rdi 4383 SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); 4384 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4385 GA->getValueType(0), 4386 GA->getOffset()); 4387 SDValue Ops[] = { DAG.getEntryNode(), TGA}; 4388 SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); 4389 Chain = Result.getValue(1); 4390 InFlag = Result.getValue(2); 4391 4392 // call __tls_get_addr. This function receives its argument in 4393 // the register RDI. 4394 Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); 4395 InFlag = Chain.getValue(1); 4396 4397 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 4398 SDValue Ops1[] = { Chain, 4399 DAG.getTargetExternalSymbol("__tls_get_addr", 4400 PtrVT), 4401 DAG.getRegister(X86::RDI, PtrVT), 4402 InFlag }; 4403 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); 4404 InFlag = Chain.getValue(1); 4405 4406 return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); 4407} 4408 4409// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 4410// "local exec" model. 4411static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 4412 const MVT PtrVT) { 4413 // Get the Thread Pointer 4414 SDValue ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); 4415 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 4416 // exec) 4417 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 4418 GA->getValueType(0), 4419 GA->getOffset()); 4420 SDValue Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); 4421 4422 if (GA->getGlobal()->isDeclaration()) // initial exec TLS model 4423 Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, 4424 PseudoSourceValue::getGOT(), 0); 4425 4426 // The address of the thread local variable is the add of the thread 4427 // pointer with the offset of the variable. 4428 return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); 4429} 4430 4431SDValue 4432X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) { 4433 // TODO: implement the "local dynamic" model 4434 // TODO: implement the "initial exec"model for pic executables 4435 assert(Subtarget->isTargetELF() && 4436 "TLS not implemented for non-ELF targets"); 4437 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 4438 // If the relocation model is PIC, use the "General Dynamic" TLS Model, 4439 // otherwise use the "Local Exec"TLS Model 4440 if (Subtarget->is64Bit()) { 4441 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 4442 } else { 4443 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) 4444 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 4445 else 4446 return LowerToTLSExecModel(GA, DAG, getPointerTy()); 4447 } 4448} 4449 4450SDValue 4451X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) { 4452 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 4453 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); 4454 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4455 // With PIC, the address is actually $g + Offset. 4456 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4457 !Subtarget->isPICStyleRIPRel()) { 4458 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4459 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4460 Result); 4461 } 4462 4463 return Result; 4464} 4465 4466SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) { 4467 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4468 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); 4469 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); 4470 // With PIC, the address is actually $g + Offset. 4471 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 4472 !Subtarget->isPICStyleRIPRel()) { 4473 Result = DAG.getNode(ISD::ADD, getPointerTy(), 4474 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), 4475 Result); 4476 } 4477 4478 return Result; 4479} 4480 4481/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 4482/// take a 2 x i32 value to shift plus a shift amount. 4483SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) { 4484 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4485 MVT VT = Op.getValueType(); 4486 unsigned VTBits = VT.getSizeInBits(); 4487 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 4488 SDValue ShOpLo = Op.getOperand(0); 4489 SDValue ShOpHi = Op.getOperand(1); 4490 SDValue ShAmt = Op.getOperand(2); 4491 SDValue Tmp1 = isSRA ? 4492 DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : 4493 DAG.getConstant(0, VT); 4494 4495 SDValue Tmp2, Tmp3; 4496 if (Op.getOpcode() == ISD::SHL_PARTS) { 4497 Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); 4498 Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); 4499 } else { 4500 Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); 4501 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); 4502 } 4503 4504 SDValue AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, 4505 DAG.getConstant(VTBits, MVT::i8)); 4506 SDValue Cond = DAG.getNode(X86ISD::CMP, VT, 4507 AndNode, DAG.getConstant(0, MVT::i8)); 4508 4509 SDValue Hi, Lo; 4510 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4511 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 4512 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 4513 4514 if (Op.getOpcode() == ISD::SHL_PARTS) { 4515 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4516 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4517 } else { 4518 Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); 4519 Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); 4520 } 4521 4522 SDValue Ops[2] = { Lo, Hi }; 4523 return DAG.getMergeValues(Ops, 2); 4524} 4525 4526SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 4527 MVT SrcVT = Op.getOperand(0).getValueType(); 4528 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 4529 "Unknown SINT_TO_FP to lower!"); 4530 4531 // These are really Legal; caller falls through into that case. 4532 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 4533 return SDValue(); 4534 if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && 4535 Subtarget->is64Bit()) 4536 return SDValue(); 4537 4538 unsigned Size = SrcVT.getSizeInBits()/8; 4539 MachineFunction &MF = DAG.getMachineFunction(); 4540 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 4541 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4542 SDValue Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), 4543 StackSlot, 4544 PseudoSourceValue::getFixedStack(SSFI), 0); 4545 4546 // Build the FILD 4547 SDVTList Tys; 4548 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 4549 if (useSSE) 4550 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 4551 else 4552 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 4553 SmallVector<SDValue, 8> Ops; 4554 Ops.push_back(Chain); 4555 Ops.push_back(StackSlot); 4556 Ops.push_back(DAG.getValueType(SrcVT)); 4557 SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, 4558 Tys, &Ops[0], Ops.size()); 4559 4560 if (useSSE) { 4561 Chain = Result.getValue(1); 4562 SDValue InFlag = Result.getValue(2); 4563 4564 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 4565 // shouldn't be necessary except that RFP cannot be live across 4566 // multiple blocks. When stackifier is fixed, they can be uncoupled. 4567 MachineFunction &MF = DAG.getMachineFunction(); 4568 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 4569 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4570 Tys = DAG.getVTList(MVT::Other); 4571 SmallVector<SDValue, 8> Ops; 4572 Ops.push_back(Chain); 4573 Ops.push_back(Result); 4574 Ops.push_back(StackSlot); 4575 Ops.push_back(DAG.getValueType(Op.getValueType())); 4576 Ops.push_back(InFlag); 4577 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); 4578 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, 4579 PseudoSourceValue::getFixedStack(SSFI), 0); 4580 } 4581 4582 return Result; 4583} 4584 4585std::pair<SDValue,SDValue> X86TargetLowering:: 4586FP_TO_SINTHelper(SDValue Op, SelectionDAG &DAG) { 4587 assert(Op.getValueType().getSimpleVT() <= MVT::i64 && 4588 Op.getValueType().getSimpleVT() >= MVT::i16 && 4589 "Unknown FP_TO_SINT to lower!"); 4590 4591 // These are really Legal. 4592 if (Op.getValueType() == MVT::i32 && 4593 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 4594 return std::make_pair(SDValue(), SDValue()); 4595 if (Subtarget->is64Bit() && 4596 Op.getValueType() == MVT::i64 && 4597 Op.getOperand(0).getValueType() != MVT::f80) 4598 return std::make_pair(SDValue(), SDValue()); 4599 4600 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 4601 // stack slot. 4602 MachineFunction &MF = DAG.getMachineFunction(); 4603 unsigned MemSize = Op.getValueType().getSizeInBits()/8; 4604 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4605 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4606 unsigned Opc; 4607 switch (Op.getValueType().getSimpleVT()) { 4608 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 4609 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 4610 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 4611 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 4612 } 4613 4614 SDValue Chain = DAG.getEntryNode(); 4615 SDValue Value = Op.getOperand(0); 4616 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 4617 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 4618 Chain = DAG.getStore(Chain, Value, StackSlot, 4619 PseudoSourceValue::getFixedStack(SSFI), 0); 4620 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 4621 SDValue Ops[] = { 4622 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 4623 }; 4624 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); 4625 Chain = Value.getValue(1); 4626 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 4627 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 4628 } 4629 4630 // Build the FP_TO_INT*_IN_MEM 4631 SDValue Ops[] = { Chain, Value, StackSlot }; 4632 SDValue FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); 4633 4634 return std::make_pair(FIST, StackSlot); 4635} 4636 4637SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) { 4638 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(Op, DAG); 4639 SDValue FIST = Vals.first, StackSlot = Vals.second; 4640 if (FIST.getNode() == 0) return SDValue(); 4641 4642 // Load the result. 4643 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); 4644} 4645 4646SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { 4647 std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG); 4648 SDValue FIST = Vals.first, StackSlot = Vals.second; 4649 if (FIST.getNode() == 0) return 0; 4650 4651 MVT VT = N->getValueType(0); 4652 4653 // Return a load from the stack slot. 4654 SDValue Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0); 4655 4656 // Use MERGE_VALUES to drop the chain result value and get a node with one 4657 // result. This requires turning off getMergeValues simplification, since 4658 // otherwise it will give us Res back. 4659 return DAG.getMergeValues(&Res, 1, false).getNode(); 4660} 4661 4662SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) { 4663 MVT VT = Op.getValueType(); 4664 MVT EltVT = VT; 4665 if (VT.isVector()) 4666 EltVT = VT.getVectorElementType(); 4667 std::vector<Constant*> CV; 4668 if (EltVT == MVT::f64) { 4669 Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); 4670 CV.push_back(C); 4671 CV.push_back(C); 4672 } else { 4673 Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); 4674 CV.push_back(C); 4675 CV.push_back(C); 4676 CV.push_back(C); 4677 CV.push_back(C); 4678 } 4679 Constant *C = ConstantVector::get(CV); 4680 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4681 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4682 PseudoSourceValue::getConstantPool(), 0, 4683 false, 16); 4684 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); 4685} 4686 4687SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) { 4688 MVT VT = Op.getValueType(); 4689 MVT EltVT = VT; 4690 unsigned EltNum = 1; 4691 if (VT.isVector()) { 4692 EltVT = VT.getVectorElementType(); 4693 EltNum = VT.getVectorNumElements(); 4694 } 4695 std::vector<Constant*> CV; 4696 if (EltVT == MVT::f64) { 4697 Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); 4698 CV.push_back(C); 4699 CV.push_back(C); 4700 } else { 4701 Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); 4702 CV.push_back(C); 4703 CV.push_back(C); 4704 CV.push_back(C); 4705 CV.push_back(C); 4706 } 4707 Constant *C = ConstantVector::get(CV); 4708 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4709 SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4710 PseudoSourceValue::getConstantPool(), 0, 4711 false, 16); 4712 if (VT.isVector()) { 4713 return DAG.getNode(ISD::BIT_CONVERT, VT, 4714 DAG.getNode(ISD::XOR, MVT::v2i64, 4715 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), 4716 DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); 4717 } else { 4718 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); 4719 } 4720} 4721 4722SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { 4723 SDValue Op0 = Op.getOperand(0); 4724 SDValue Op1 = Op.getOperand(1); 4725 MVT VT = Op.getValueType(); 4726 MVT SrcVT = Op1.getValueType(); 4727 4728 // If second operand is smaller, extend it first. 4729 if (SrcVT.bitsLT(VT)) { 4730 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); 4731 SrcVT = VT; 4732 } 4733 // And if it is bigger, shrink it first. 4734 if (SrcVT.bitsGT(VT)) { 4735 Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); 4736 SrcVT = VT; 4737 } 4738 4739 // At this point the operands and the result should have the same 4740 // type, and that won't be f80 since that is not custom lowered. 4741 4742 // First get the sign bit of second operand. 4743 std::vector<Constant*> CV; 4744 if (SrcVT == MVT::f64) { 4745 CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); 4746 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4747 } else { 4748 CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); 4749 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4750 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4751 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4752 } 4753 Constant *C = ConstantVector::get(CV); 4754 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4755 SDValue Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, 4756 PseudoSourceValue::getConstantPool(), 0, 4757 false, 16); 4758 SDValue SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); 4759 4760 // Shift sign bit right or left if the two operands have different types. 4761 if (SrcVT.bitsGT(VT)) { 4762 // Op0 is MVT::f32, Op1 is MVT::f64. 4763 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); 4764 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, 4765 DAG.getConstant(32, MVT::i32)); 4766 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); 4767 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, 4768 DAG.getIntPtrConstant(0)); 4769 } 4770 4771 // Clear first operand sign bit. 4772 CV.clear(); 4773 if (VT == MVT::f64) { 4774 CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); 4775 CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); 4776 } else { 4777 CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); 4778 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4779 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4780 CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); 4781 } 4782 C = ConstantVector::get(CV); 4783 CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); 4784 SDValue Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 4785 PseudoSourceValue::getConstantPool(), 0, 4786 false, 16); 4787 SDValue Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); 4788 4789 // Or the value with the sign bit. 4790 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); 4791} 4792 4793SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) { 4794 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 4795 SDValue Cond; 4796 SDValue Op0 = Op.getOperand(0); 4797 SDValue Op1 = Op.getOperand(1); 4798 SDValue CC = Op.getOperand(2); 4799 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4800 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4801 unsigned X86CC; 4802 4803 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, 4804 Op0, Op1, DAG)) { 4805 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4806 return DAG.getNode(X86ISD::SETCC, MVT::i8, 4807 DAG.getConstant(X86CC, MVT::i8), Cond); 4808 } 4809 4810 assert(isFP && "Illegal integer SetCC!"); 4811 4812 Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); 4813 switch (SetCCOpcode) { 4814 default: assert(false && "Illegal floating point SetCC!"); 4815 case ISD::SETOEQ: { // !PF & ZF 4816 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4817 DAG.getConstant(X86::COND_NP, MVT::i8), Cond); 4818 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4819 DAG.getConstant(X86::COND_E, MVT::i8), Cond); 4820 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 4821 } 4822 case ISD::SETUNE: { // PF | !ZF 4823 SDValue Tmp1 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4824 DAG.getConstant(X86::COND_P, MVT::i8), Cond); 4825 SDValue Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 4826 DAG.getConstant(X86::COND_NE, MVT::i8), Cond); 4827 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 4828 } 4829 } 4830} 4831 4832SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 4833 SDValue Cond; 4834 SDValue Op0 = Op.getOperand(0); 4835 SDValue Op1 = Op.getOperand(1); 4836 SDValue CC = Op.getOperand(2); 4837 MVT VT = Op.getValueType(); 4838 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 4839 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 4840 4841 if (isFP) { 4842 unsigned SSECC = 8; 4843 MVT VT0 = Op0.getValueType(); 4844 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); 4845 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; 4846 bool Swap = false; 4847 4848 switch (SetCCOpcode) { 4849 default: break; 4850 case ISD::SETOEQ: 4851 case ISD::SETEQ: SSECC = 0; break; 4852 case ISD::SETOGT: 4853 case ISD::SETGT: Swap = true; // Fallthrough 4854 case ISD::SETLT: 4855 case ISD::SETOLT: SSECC = 1; break; 4856 case ISD::SETOGE: 4857 case ISD::SETGE: Swap = true; // Fallthrough 4858 case ISD::SETLE: 4859 case ISD::SETOLE: SSECC = 2; break; 4860 case ISD::SETUO: SSECC = 3; break; 4861 case ISD::SETUNE: 4862 case ISD::SETNE: SSECC = 4; break; 4863 case ISD::SETULE: Swap = true; 4864 case ISD::SETUGE: SSECC = 5; break; 4865 case ISD::SETULT: Swap = true; 4866 case ISD::SETUGT: SSECC = 6; break; 4867 case ISD::SETO: SSECC = 7; break; 4868 } 4869 if (Swap) 4870 std::swap(Op0, Op1); 4871 4872 // In the two special cases we can't handle, emit two comparisons. 4873 if (SSECC == 8) { 4874 if (SetCCOpcode == ISD::SETUEQ) { 4875 SDValue UNORD, EQ; 4876 UNORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 4877 EQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 4878 return DAG.getNode(ISD::OR, VT, UNORD, EQ); 4879 } 4880 else if (SetCCOpcode == ISD::SETONE) { 4881 SDValue ORD, NEQ; 4882 ORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 4883 NEQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 4884 return DAG.getNode(ISD::AND, VT, ORD, NEQ); 4885 } 4886 assert(0 && "Illegal FP comparison"); 4887 } 4888 // Handle all other FP comparisons here. 4889 return DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 4890 } 4891 4892 // We are handling one of the integer comparisons here. Since SSE only has 4893 // GT and EQ comparisons for integer, swapping operands and multiple 4894 // operations may be required for some comparisons. 4895 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 4896 bool Swap = false, Invert = false, FlipSigns = false; 4897 4898 switch (VT.getSimpleVT()) { 4899 default: break; 4900 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 4901 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 4902 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 4903 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 4904 } 4905 4906 switch (SetCCOpcode) { 4907 default: break; 4908 case ISD::SETNE: Invert = true; 4909 case ISD::SETEQ: Opc = EQOpc; break; 4910 case ISD::SETLT: Swap = true; 4911 case ISD::SETGT: Opc = GTOpc; break; 4912 case ISD::SETGE: Swap = true; 4913 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 4914 case ISD::SETULT: Swap = true; 4915 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 4916 case ISD::SETUGE: Swap = true; 4917 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 4918 } 4919 if (Swap) 4920 std::swap(Op0, Op1); 4921 4922 // Since SSE has no unsigned integer comparisons, we need to flip the sign 4923 // bits of the inputs before performing those operations. 4924 if (FlipSigns) { 4925 MVT EltVT = VT.getVectorElementType(); 4926 SDValue SignBit = DAG.getConstant(EltVT.getIntegerVTSignBit(), EltVT); 4927 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 4928 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, VT, &SignBits[0], 4929 SignBits.size()); 4930 Op0 = DAG.getNode(ISD::XOR, VT, Op0, SignVec); 4931 Op1 = DAG.getNode(ISD::XOR, VT, Op1, SignVec); 4932 } 4933 4934 SDValue Result = DAG.getNode(Opc, VT, Op0, Op1); 4935 4936 // If the logical-not of the result is required, perform that now. 4937 if (Invert) { 4938 MVT EltVT = VT.getVectorElementType(); 4939 SDValue NegOne = DAG.getConstant(EltVT.getIntegerVTBitMask(), EltVT); 4940 std::vector<SDValue> NegOnes(VT.getVectorNumElements(), NegOne); 4941 SDValue NegOneV = DAG.getNode(ISD::BUILD_VECTOR, VT, &NegOnes[0], 4942 NegOnes.size()); 4943 Result = DAG.getNode(ISD::XOR, VT, Result, NegOneV); 4944 } 4945 return Result; 4946} 4947 4948SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) { 4949 bool addTest = true; 4950 SDValue Cond = Op.getOperand(0); 4951 SDValue CC; 4952 4953 if (Cond.getOpcode() == ISD::SETCC) 4954 Cond = LowerSETCC(Cond, DAG); 4955 4956 // If condition flag is set by a X86ISD::CMP, then use it as the condition 4957 // setting operand in place of the X86ISD::SETCC. 4958 if (Cond.getOpcode() == X86ISD::SETCC) { 4959 CC = Cond.getOperand(0); 4960 4961 SDValue Cmp = Cond.getOperand(1); 4962 unsigned Opc = Cmp.getOpcode(); 4963 MVT VT = Op.getValueType(); 4964 4965 bool IllegalFPCMov = false; 4966 if (VT.isFloatingPoint() && !VT.isVector() && 4967 !isScalarFPTypeInSSEReg(VT)) // FPStack? 4968 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 4969 4970 if ((Opc == X86ISD::CMP || 4971 Opc == X86ISD::COMI || 4972 Opc == X86ISD::UCOMI) && !IllegalFPCMov) { 4973 Cond = Cmp; 4974 addTest = false; 4975 } 4976 } 4977 4978 if (addTest) { 4979 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 4980 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 4981 } 4982 4983 const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), 4984 MVT::Flag); 4985 SmallVector<SDValue, 4> Ops; 4986 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 4987 // condition is true. 4988 Ops.push_back(Op.getOperand(2)); 4989 Ops.push_back(Op.getOperand(1)); 4990 Ops.push_back(CC); 4991 Ops.push_back(Cond); 4992 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); 4993} 4994 4995SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) { 4996 bool addTest = true; 4997 SDValue Chain = Op.getOperand(0); 4998 SDValue Cond = Op.getOperand(1); 4999 SDValue Dest = Op.getOperand(2); 5000 SDValue CC; 5001 5002 if (Cond.getOpcode() == ISD::SETCC) 5003 Cond = LowerSETCC(Cond, DAG); 5004 5005 // If condition flag is set by a X86ISD::CMP, then use it as the condition 5006 // setting operand in place of the X86ISD::SETCC. 5007 if (Cond.getOpcode() == X86ISD::SETCC) { 5008 CC = Cond.getOperand(0); 5009 5010 SDValue Cmp = Cond.getOperand(1); 5011 unsigned Opc = Cmp.getOpcode(); 5012 if (Opc == X86ISD::CMP || 5013 Opc == X86ISD::COMI || 5014 Opc == X86ISD::UCOMI) { 5015 Cond = Cmp; 5016 addTest = false; 5017 } 5018 } 5019 5020 if (addTest) { 5021 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 5022 Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); 5023 } 5024 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 5025 Chain, Op.getOperand(2), CC, Cond); 5026} 5027 5028 5029// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 5030// Calls to _alloca is needed to probe the stack when allocating more than 4k 5031// bytes in one go. Touching the stack at 4K increments is necessary to ensure 5032// that the guard pages used by the OS virtual memory manager are allocated in 5033// correct sequence. 5034SDValue 5035X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 5036 SelectionDAG &DAG) { 5037 assert(Subtarget->isTargetCygMing() && 5038 "This should be used only on Cygwin/Mingw targets"); 5039 5040 // Get the inputs. 5041 SDValue Chain = Op.getOperand(0); 5042 SDValue Size = Op.getOperand(1); 5043 // FIXME: Ensure alignment here 5044 5045 SDValue Flag; 5046 5047 MVT IntPtr = getPointerTy(); 5048 MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 5049 5050 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0)); 5051 5052 Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); 5053 Flag = Chain.getValue(1); 5054 5055 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 5056 SDValue Ops[] = { Chain, 5057 DAG.getTargetExternalSymbol("_alloca", IntPtr), 5058 DAG.getRegister(X86::EAX, IntPtr), 5059 DAG.getRegister(X86StackPtr, SPTy), 5060 Flag }; 5061 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); 5062 Flag = Chain.getValue(1); 5063 5064 Chain = DAG.getCALLSEQ_END(Chain, 5065 DAG.getIntPtrConstant(0), 5066 DAG.getIntPtrConstant(0), 5067 Flag); 5068 5069 Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); 5070 5071 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 5072 return DAG.getMergeValues(Ops1, 2); 5073} 5074 5075SDValue 5076X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, 5077 SDValue Chain, 5078 SDValue Dst, SDValue Src, 5079 SDValue Size, unsigned Align, 5080 const Value *DstSV, uint64_t DstSVOff) { 5081 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5082 5083 /// If not DWORD aligned or size is more than the threshold, call the library. 5084 /// The libc version is likely to be faster for these cases. It can use the 5085 /// address value and run time information about the CPU. 5086 if ((Align & 3) != 0 || 5087 !ConstantSize || 5088 ConstantSize->getValue() > getSubtarget()->getMaxInlineSizeThreshold()) { 5089 SDValue InFlag(0, 0); 5090 5091 // Check to see if there is a specialized entry-point for memory zeroing. 5092 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 5093 if (const char *bzeroEntry = 5094 V && V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 5095 MVT IntPtr = getPointerTy(); 5096 const Type *IntPtrTy = getTargetData()->getIntPtrType(); 5097 TargetLowering::ArgListTy Args; 5098 TargetLowering::ArgListEntry Entry; 5099 Entry.Node = Dst; 5100 Entry.Ty = IntPtrTy; 5101 Args.push_back(Entry); 5102 Entry.Node = Size; 5103 Args.push_back(Entry); 5104 std::pair<SDValue,SDValue> CallResult = 5105 LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C, 5106 false, DAG.getExternalSymbol(bzeroEntry, IntPtr), 5107 Args, DAG); 5108 return CallResult.second; 5109 } 5110 5111 // Otherwise have the target-independent code call memset. 5112 return SDValue(); 5113 } 5114 5115 uint64_t SizeVal = ConstantSize->getValue(); 5116 SDValue InFlag(0, 0); 5117 MVT AVT; 5118 SDValue Count; 5119 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 5120 unsigned BytesLeft = 0; 5121 bool TwoRepStos = false; 5122 if (ValC) { 5123 unsigned ValReg; 5124 uint64_t Val = ValC->getValue() & 255; 5125 5126 // If the value is a constant, then we can potentially use larger sets. 5127 switch (Align & 3) { 5128 case 2: // WORD aligned 5129 AVT = MVT::i16; 5130 ValReg = X86::AX; 5131 Val = (Val << 8) | Val; 5132 break; 5133 case 0: // DWORD aligned 5134 AVT = MVT::i32; 5135 ValReg = X86::EAX; 5136 Val = (Val << 8) | Val; 5137 Val = (Val << 16) | Val; 5138 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 5139 AVT = MVT::i64; 5140 ValReg = X86::RAX; 5141 Val = (Val << 32) | Val; 5142 } 5143 break; 5144 default: // Byte aligned 5145 AVT = MVT::i8; 5146 ValReg = X86::AL; 5147 Count = DAG.getIntPtrConstant(SizeVal); 5148 break; 5149 } 5150 5151 if (AVT.bitsGT(MVT::i8)) { 5152 unsigned UBytes = AVT.getSizeInBits() / 8; 5153 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 5154 BytesLeft = SizeVal % UBytes; 5155 } 5156 5157 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 5158 InFlag); 5159 InFlag = Chain.getValue(1); 5160 } else { 5161 AVT = MVT::i8; 5162 Count = DAG.getIntPtrConstant(SizeVal); 5163 Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); 5164 InFlag = Chain.getValue(1); 5165 } 5166 5167 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5168 Count, InFlag); 5169 InFlag = Chain.getValue(1); 5170 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5171 Dst, InFlag); 5172 InFlag = Chain.getValue(1); 5173 5174 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5175 SmallVector<SDValue, 8> Ops; 5176 Ops.push_back(Chain); 5177 Ops.push_back(DAG.getValueType(AVT)); 5178 Ops.push_back(InFlag); 5179 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5180 5181 if (TwoRepStos) { 5182 InFlag = Chain.getValue(1); 5183 Count = Size; 5184 MVT CVT = Count.getValueType(); 5185 SDValue Left = DAG.getNode(ISD::AND, CVT, Count, 5186 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 5187 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, 5188 Left, InFlag); 5189 InFlag = Chain.getValue(1); 5190 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5191 Ops.clear(); 5192 Ops.push_back(Chain); 5193 Ops.push_back(DAG.getValueType(MVT::i8)); 5194 Ops.push_back(InFlag); 5195 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); 5196 } else if (BytesLeft) { 5197 // Handle the last 1 - 7 bytes. 5198 unsigned Offset = SizeVal - BytesLeft; 5199 MVT AddrVT = Dst.getValueType(); 5200 MVT SizeVT = Size.getValueType(); 5201 5202 Chain = DAG.getMemset(Chain, 5203 DAG.getNode(ISD::ADD, AddrVT, Dst, 5204 DAG.getConstant(Offset, AddrVT)), 5205 Src, 5206 DAG.getConstant(BytesLeft, SizeVT), 5207 Align, DstSV, DstSVOff + Offset); 5208 } 5209 5210 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 5211 return Chain; 5212} 5213 5214SDValue 5215X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, 5216 SDValue Chain, SDValue Dst, SDValue Src, 5217 SDValue Size, unsigned Align, 5218 bool AlwaysInline, 5219 const Value *DstSV, uint64_t DstSVOff, 5220 const Value *SrcSV, uint64_t SrcSVOff) { 5221 // This requires the copy size to be a constant, preferrably 5222 // within a subtarget-specific limit. 5223 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5224 if (!ConstantSize) 5225 return SDValue(); 5226 uint64_t SizeVal = ConstantSize->getValue(); 5227 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 5228 return SDValue(); 5229 5230 /// If not DWORD aligned, call the library. 5231 if ((Align & 3) != 0) 5232 return SDValue(); 5233 5234 // DWORD aligned 5235 MVT AVT = MVT::i32; 5236 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) // QWORD aligned 5237 AVT = MVT::i64; 5238 5239 unsigned UBytes = AVT.getSizeInBits() / 8; 5240 unsigned CountVal = SizeVal / UBytes; 5241 SDValue Count = DAG.getIntPtrConstant(CountVal); 5242 unsigned BytesLeft = SizeVal % UBytes; 5243 5244 SDValue InFlag(0, 0); 5245 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, 5246 Count, InFlag); 5247 InFlag = Chain.getValue(1); 5248 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, 5249 Dst, InFlag); 5250 InFlag = Chain.getValue(1); 5251 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, 5252 Src, InFlag); 5253 InFlag = Chain.getValue(1); 5254 5255 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5256 SmallVector<SDValue, 8> Ops; 5257 Ops.push_back(Chain); 5258 Ops.push_back(DAG.getValueType(AVT)); 5259 Ops.push_back(InFlag); 5260 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); 5261 5262 SmallVector<SDValue, 4> Results; 5263 Results.push_back(RepMovs); 5264 if (BytesLeft) { 5265 // Handle the last 1 - 7 bytes. 5266 unsigned Offset = SizeVal - BytesLeft; 5267 MVT DstVT = Dst.getValueType(); 5268 MVT SrcVT = Src.getValueType(); 5269 MVT SizeVT = Size.getValueType(); 5270 Results.push_back(DAG.getMemcpy(Chain, 5271 DAG.getNode(ISD::ADD, DstVT, Dst, 5272 DAG.getConstant(Offset, DstVT)), 5273 DAG.getNode(ISD::ADD, SrcVT, Src, 5274 DAG.getConstant(Offset, SrcVT)), 5275 DAG.getConstant(BytesLeft, SizeVT), 5276 Align, AlwaysInline, 5277 DstSV, DstSVOff + Offset, 5278 SrcSV, SrcSVOff + Offset)); 5279 } 5280 5281 return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); 5282} 5283 5284/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain 5285SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ 5286 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5287 SDValue TheChain = N->getOperand(0); 5288 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); 5289 if (Subtarget->is64Bit()) { 5290 SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); 5291 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, 5292 MVT::i64, rax.getValue(2)); 5293 SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, 5294 DAG.getConstant(32, MVT::i8)); 5295 SDValue Ops[] = { 5296 DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) 5297 }; 5298 5299 return DAG.getMergeValues(Ops, 2).getNode(); 5300 } 5301 5302 SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); 5303 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, 5304 MVT::i32, eax.getValue(2)); 5305 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 5306 SDValue Ops[] = { eax, edx }; 5307 Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); 5308 5309 // Use a MERGE_VALUES to return the value and chain. 5310 Ops[1] = edx.getValue(1); 5311 return DAG.getMergeValues(Ops, 2).getNode(); 5312} 5313 5314SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) { 5315 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 5316 5317 if (!Subtarget->is64Bit()) { 5318 // vastart just stores the address of the VarArgsFrameIndex slot into the 5319 // memory location argument. 5320 SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5321 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); 5322 } 5323 5324 // __va_list_tag: 5325 // gp_offset (0 - 6 * 8) 5326 // fp_offset (48 - 48 + 8 * 16) 5327 // overflow_arg_area (point to parameters coming in memory). 5328 // reg_save_area 5329 SmallVector<SDValue, 8> MemOps; 5330 SDValue FIN = Op.getOperand(1); 5331 // Store gp_offset 5332 SDValue Store = DAG.getStore(Op.getOperand(0), 5333 DAG.getConstant(VarArgsGPOffset, MVT::i32), 5334 FIN, SV, 0); 5335 MemOps.push_back(Store); 5336 5337 // Store fp_offset 5338 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5339 Store = DAG.getStore(Op.getOperand(0), 5340 DAG.getConstant(VarArgsFPOffset, MVT::i32), 5341 FIN, SV, 0); 5342 MemOps.push_back(Store); 5343 5344 // Store ptr to overflow_arg_area 5345 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); 5346 SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 5347 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); 5348 MemOps.push_back(Store); 5349 5350 // Store ptr to reg_save_area. 5351 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); 5352 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 5353 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); 5354 MemOps.push_back(Store); 5355 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); 5356} 5357 5358SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) { 5359 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5360 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 5361 SDValue Chain = Op.getOperand(0); 5362 SDValue SrcPtr = Op.getOperand(1); 5363 SDValue SrcSV = Op.getOperand(2); 5364 5365 assert(0 && "VAArgInst is not yet implemented for x86-64!"); 5366 abort(); 5367 return SDValue(); 5368} 5369 5370SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) { 5371 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 5372 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 5373 SDValue Chain = Op.getOperand(0); 5374 SDValue DstPtr = Op.getOperand(1); 5375 SDValue SrcPtr = Op.getOperand(2); 5376 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 5377 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5378 5379 return DAG.getMemcpy(Chain, DstPtr, SrcPtr, 5380 DAG.getIntPtrConstant(24), 8, false, 5381 DstSV, 0, SrcSV, 0); 5382} 5383 5384SDValue 5385X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { 5386 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); 5387 switch (IntNo) { 5388 default: return SDValue(); // Don't custom lower most intrinsics. 5389 // Comparison intrinsics. 5390 case Intrinsic::x86_sse_comieq_ss: 5391 case Intrinsic::x86_sse_comilt_ss: 5392 case Intrinsic::x86_sse_comile_ss: 5393 case Intrinsic::x86_sse_comigt_ss: 5394 case Intrinsic::x86_sse_comige_ss: 5395 case Intrinsic::x86_sse_comineq_ss: 5396 case Intrinsic::x86_sse_ucomieq_ss: 5397 case Intrinsic::x86_sse_ucomilt_ss: 5398 case Intrinsic::x86_sse_ucomile_ss: 5399 case Intrinsic::x86_sse_ucomigt_ss: 5400 case Intrinsic::x86_sse_ucomige_ss: 5401 case Intrinsic::x86_sse_ucomineq_ss: 5402 case Intrinsic::x86_sse2_comieq_sd: 5403 case Intrinsic::x86_sse2_comilt_sd: 5404 case Intrinsic::x86_sse2_comile_sd: 5405 case Intrinsic::x86_sse2_comigt_sd: 5406 case Intrinsic::x86_sse2_comige_sd: 5407 case Intrinsic::x86_sse2_comineq_sd: 5408 case Intrinsic::x86_sse2_ucomieq_sd: 5409 case Intrinsic::x86_sse2_ucomilt_sd: 5410 case Intrinsic::x86_sse2_ucomile_sd: 5411 case Intrinsic::x86_sse2_ucomigt_sd: 5412 case Intrinsic::x86_sse2_ucomige_sd: 5413 case Intrinsic::x86_sse2_ucomineq_sd: { 5414 unsigned Opc = 0; 5415 ISD::CondCode CC = ISD::SETCC_INVALID; 5416 switch (IntNo) { 5417 default: break; 5418 case Intrinsic::x86_sse_comieq_ss: 5419 case Intrinsic::x86_sse2_comieq_sd: 5420 Opc = X86ISD::COMI; 5421 CC = ISD::SETEQ; 5422 break; 5423 case Intrinsic::x86_sse_comilt_ss: 5424 case Intrinsic::x86_sse2_comilt_sd: 5425 Opc = X86ISD::COMI; 5426 CC = ISD::SETLT; 5427 break; 5428 case Intrinsic::x86_sse_comile_ss: 5429 case Intrinsic::x86_sse2_comile_sd: 5430 Opc = X86ISD::COMI; 5431 CC = ISD::SETLE; 5432 break; 5433 case Intrinsic::x86_sse_comigt_ss: 5434 case Intrinsic::x86_sse2_comigt_sd: 5435 Opc = X86ISD::COMI; 5436 CC = ISD::SETGT; 5437 break; 5438 case Intrinsic::x86_sse_comige_ss: 5439 case Intrinsic::x86_sse2_comige_sd: 5440 Opc = X86ISD::COMI; 5441 CC = ISD::SETGE; 5442 break; 5443 case Intrinsic::x86_sse_comineq_ss: 5444 case Intrinsic::x86_sse2_comineq_sd: 5445 Opc = X86ISD::COMI; 5446 CC = ISD::SETNE; 5447 break; 5448 case Intrinsic::x86_sse_ucomieq_ss: 5449 case Intrinsic::x86_sse2_ucomieq_sd: 5450 Opc = X86ISD::UCOMI; 5451 CC = ISD::SETEQ; 5452 break; 5453 case Intrinsic::x86_sse_ucomilt_ss: 5454 case Intrinsic::x86_sse2_ucomilt_sd: 5455 Opc = X86ISD::UCOMI; 5456 CC = ISD::SETLT; 5457 break; 5458 case Intrinsic::x86_sse_ucomile_ss: 5459 case Intrinsic::x86_sse2_ucomile_sd: 5460 Opc = X86ISD::UCOMI; 5461 CC = ISD::SETLE; 5462 break; 5463 case Intrinsic::x86_sse_ucomigt_ss: 5464 case Intrinsic::x86_sse2_ucomigt_sd: 5465 Opc = X86ISD::UCOMI; 5466 CC = ISD::SETGT; 5467 break; 5468 case Intrinsic::x86_sse_ucomige_ss: 5469 case Intrinsic::x86_sse2_ucomige_sd: 5470 Opc = X86ISD::UCOMI; 5471 CC = ISD::SETGE; 5472 break; 5473 case Intrinsic::x86_sse_ucomineq_ss: 5474 case Intrinsic::x86_sse2_ucomineq_sd: 5475 Opc = X86ISD::UCOMI; 5476 CC = ISD::SETNE; 5477 break; 5478 } 5479 5480 unsigned X86CC; 5481 SDValue LHS = Op.getOperand(1); 5482 SDValue RHS = Op.getOperand(2); 5483 translateX86CC(CC, true, X86CC, LHS, RHS, DAG); 5484 5485 SDValue Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); 5486 SDValue SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, 5487 DAG.getConstant(X86CC, MVT::i8), Cond); 5488 return DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, SetCC); 5489 } 5490 5491 // Fix vector shift instructions where the last operand is a non-immediate 5492 // i32 value. 5493 case Intrinsic::x86_sse2_pslli_w: 5494 case Intrinsic::x86_sse2_pslli_d: 5495 case Intrinsic::x86_sse2_pslli_q: 5496 case Intrinsic::x86_sse2_psrli_w: 5497 case Intrinsic::x86_sse2_psrli_d: 5498 case Intrinsic::x86_sse2_psrli_q: 5499 case Intrinsic::x86_sse2_psrai_w: 5500 case Intrinsic::x86_sse2_psrai_d: 5501 case Intrinsic::x86_mmx_pslli_w: 5502 case Intrinsic::x86_mmx_pslli_d: 5503 case Intrinsic::x86_mmx_pslli_q: 5504 case Intrinsic::x86_mmx_psrli_w: 5505 case Intrinsic::x86_mmx_psrli_d: 5506 case Intrinsic::x86_mmx_psrli_q: 5507 case Intrinsic::x86_mmx_psrai_w: 5508 case Intrinsic::x86_mmx_psrai_d: { 5509 SDValue ShAmt = Op.getOperand(2); 5510 if (isa<ConstantSDNode>(ShAmt)) 5511 return SDValue(); 5512 5513 unsigned NewIntNo = 0; 5514 MVT ShAmtVT = MVT::v4i32; 5515 switch (IntNo) { 5516 case Intrinsic::x86_sse2_pslli_w: 5517 NewIntNo = Intrinsic::x86_sse2_psll_w; 5518 break; 5519 case Intrinsic::x86_sse2_pslli_d: 5520 NewIntNo = Intrinsic::x86_sse2_psll_d; 5521 break; 5522 case Intrinsic::x86_sse2_pslli_q: 5523 NewIntNo = Intrinsic::x86_sse2_psll_q; 5524 break; 5525 case Intrinsic::x86_sse2_psrli_w: 5526 NewIntNo = Intrinsic::x86_sse2_psrl_w; 5527 break; 5528 case Intrinsic::x86_sse2_psrli_d: 5529 NewIntNo = Intrinsic::x86_sse2_psrl_d; 5530 break; 5531 case Intrinsic::x86_sse2_psrli_q: 5532 NewIntNo = Intrinsic::x86_sse2_psrl_q; 5533 break; 5534 case Intrinsic::x86_sse2_psrai_w: 5535 NewIntNo = Intrinsic::x86_sse2_psra_w; 5536 break; 5537 case Intrinsic::x86_sse2_psrai_d: 5538 NewIntNo = Intrinsic::x86_sse2_psra_d; 5539 break; 5540 default: { 5541 ShAmtVT = MVT::v2i32; 5542 switch (IntNo) { 5543 case Intrinsic::x86_mmx_pslli_w: 5544 NewIntNo = Intrinsic::x86_mmx_psll_w; 5545 break; 5546 case Intrinsic::x86_mmx_pslli_d: 5547 NewIntNo = Intrinsic::x86_mmx_psll_d; 5548 break; 5549 case Intrinsic::x86_mmx_pslli_q: 5550 NewIntNo = Intrinsic::x86_mmx_psll_q; 5551 break; 5552 case Intrinsic::x86_mmx_psrli_w: 5553 NewIntNo = Intrinsic::x86_mmx_psrl_w; 5554 break; 5555 case Intrinsic::x86_mmx_psrli_d: 5556 NewIntNo = Intrinsic::x86_mmx_psrl_d; 5557 break; 5558 case Intrinsic::x86_mmx_psrli_q: 5559 NewIntNo = Intrinsic::x86_mmx_psrl_q; 5560 break; 5561 case Intrinsic::x86_mmx_psrai_w: 5562 NewIntNo = Intrinsic::x86_mmx_psra_w; 5563 break; 5564 case Intrinsic::x86_mmx_psrai_d: 5565 NewIntNo = Intrinsic::x86_mmx_psra_d; 5566 break; 5567 default: abort(); // Can't reach here. 5568 } 5569 break; 5570 } 5571 } 5572 MVT VT = Op.getValueType(); 5573 ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, 5574 DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); 5575 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, 5576 DAG.getConstant(NewIntNo, MVT::i32), 5577 Op.getOperand(1), ShAmt); 5578 } 5579 } 5580} 5581 5582SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) { 5583 // Depths > 0 not supported yet! 5584 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5585 return SDValue(); 5586 5587 // Just load the return address 5588 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5589 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); 5590} 5591 5592SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) { 5593 // Depths > 0 not supported yet! 5594 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0) 5595 return SDValue(); 5596 5597 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 5598 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, 5599 DAG.getIntPtrConstant(!Subtarget->is64Bit() ? 4 : 8)); 5600} 5601 5602SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 5603 SelectionDAG &DAG) { 5604 return DAG.getIntPtrConstant(Subtarget->is64Bit() ? 16 : 8); 5605} 5606 5607SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) 5608{ 5609 assert(!Subtarget->is64Bit() && 5610 "Lowering of eh_return builtin is not supported yet on x86-64"); 5611 5612 MachineFunction &MF = DAG.getMachineFunction(); 5613 SDValue Chain = Op.getOperand(0); 5614 SDValue Offset = Op.getOperand(1); 5615 SDValue Handler = Op.getOperand(2); 5616 5617 SDValue Frame = DAG.getRegister(RegInfo->getFrameRegister(MF), 5618 getPointerTy()); 5619 5620 SDValue StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, 5621 DAG.getIntPtrConstant(-4UL)); 5622 StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); 5623 Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); 5624 Chain = DAG.getCopyToReg(Chain, X86::ECX, StoreAddr); 5625 MF.getRegInfo().addLiveOut(X86::ECX); 5626 5627 return DAG.getNode(X86ISD::EH_RETURN, MVT::Other, 5628 Chain, DAG.getRegister(X86::ECX, getPointerTy())); 5629} 5630 5631SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, 5632 SelectionDAG &DAG) { 5633 SDValue Root = Op.getOperand(0); 5634 SDValue Trmp = Op.getOperand(1); // trampoline 5635 SDValue FPtr = Op.getOperand(2); // nested function 5636 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 5637 5638 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 5639 5640 const X86InstrInfo *TII = 5641 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 5642 5643 if (Subtarget->is64Bit()) { 5644 SDValue OutChains[6]; 5645 5646 // Large code-model. 5647 5648 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 5649 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 5650 5651 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 5652 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 5653 5654 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 5655 5656 // Load the pointer to the nested function into R11. 5657 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 5658 SDValue Addr = Trmp; 5659 OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5660 TrmpAddr, 0); 5661 5662 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); 5663 OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); 5664 5665 // Load the 'nest' parameter value into R10. 5666 // R10 is specified in X86CallingConv.td 5667 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 5668 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); 5669 OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5670 TrmpAddr, 10); 5671 5672 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); 5673 OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); 5674 5675 // Jump to the nested function. 5676 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 5677 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); 5678 OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, 5679 TrmpAddr, 20); 5680 5681 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 5682 Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); 5683 OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, 5684 TrmpAddr, 22); 5685 5686 SDValue Ops[] = 5687 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; 5688 return DAG.getMergeValues(Ops, 2); 5689 } else { 5690 const Function *Func = 5691 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 5692 unsigned CC = Func->getCallingConv(); 5693 unsigned NestReg; 5694 5695 switch (CC) { 5696 default: 5697 assert(0 && "Unsupported calling convention"); 5698 case CallingConv::C: 5699 case CallingConv::X86_StdCall: { 5700 // Pass 'nest' parameter in ECX. 5701 // Must be kept in sync with X86CallingConv.td 5702 NestReg = X86::ECX; 5703 5704 // Check that ECX wasn't needed by an 'inreg' parameter. 5705 const FunctionType *FTy = Func->getFunctionType(); 5706 const PAListPtr &Attrs = Func->getParamAttrs(); 5707 5708 if (!Attrs.isEmpty() && !Func->isVarArg()) { 5709 unsigned InRegCount = 0; 5710 unsigned Idx = 1; 5711 5712 for (FunctionType::param_iterator I = FTy->param_begin(), 5713 E = FTy->param_end(); I != E; ++I, ++Idx) 5714 if (Attrs.paramHasAttr(Idx, ParamAttr::InReg)) 5715 // FIXME: should only count parameters that are lowered to integers. 5716 InRegCount += (getTargetData()->getTypeSizeInBits(*I) + 31) / 32; 5717 5718 if (InRegCount > 2) { 5719 cerr << "Nest register in use - reduce number of inreg parameters!\n"; 5720 abort(); 5721 } 5722 } 5723 break; 5724 } 5725 case CallingConv::X86_FastCall: 5726 // Pass 'nest' parameter in EAX. 5727 // Must be kept in sync with X86CallingConv.td 5728 NestReg = X86::EAX; 5729 break; 5730 } 5731 5732 SDValue OutChains[4]; 5733 SDValue Addr, Disp; 5734 5735 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); 5736 Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); 5737 5738 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 5739 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 5740 OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 5741 Trmp, TrmpAddr, 0); 5742 5743 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); 5744 OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); 5745 5746 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 5747 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); 5748 OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, 5749 TrmpAddr, 5, false, 1); 5750 5751 Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); 5752 OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); 5753 5754 SDValue Ops[] = 5755 { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; 5756 return DAG.getMergeValues(Ops, 2); 5757 } 5758} 5759 5760SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) { 5761 /* 5762 The rounding mode is in bits 11:10 of FPSR, and has the following 5763 settings: 5764 00 Round to nearest 5765 01 Round to -inf 5766 10 Round to +inf 5767 11 Round to 0 5768 5769 FLT_ROUNDS, on the other hand, expects the following: 5770 -1 Undefined 5771 0 Round to 0 5772 1 Round to nearest 5773 2 Round to +inf 5774 3 Round to -inf 5775 5776 To perform the conversion, we do: 5777 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 5778 */ 5779 5780 MachineFunction &MF = DAG.getMachineFunction(); 5781 const TargetMachine &TM = MF.getTarget(); 5782 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 5783 unsigned StackAlignment = TFI.getStackAlignment(); 5784 MVT VT = Op.getValueType(); 5785 5786 // Save FP Control Word to stack slot 5787 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 5788 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 5789 5790 SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, 5791 DAG.getEntryNode(), StackSlot); 5792 5793 // Load FP Control Word from stack slot 5794 SDValue CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); 5795 5796 // Transform as necessary 5797 SDValue CWD1 = 5798 DAG.getNode(ISD::SRL, MVT::i16, 5799 DAG.getNode(ISD::AND, MVT::i16, 5800 CWD, DAG.getConstant(0x800, MVT::i16)), 5801 DAG.getConstant(11, MVT::i8)); 5802 SDValue CWD2 = 5803 DAG.getNode(ISD::SRL, MVT::i16, 5804 DAG.getNode(ISD::AND, MVT::i16, 5805 CWD, DAG.getConstant(0x400, MVT::i16)), 5806 DAG.getConstant(9, MVT::i8)); 5807 5808 SDValue RetVal = 5809 DAG.getNode(ISD::AND, MVT::i16, 5810 DAG.getNode(ISD::ADD, MVT::i16, 5811 DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), 5812 DAG.getConstant(1, MVT::i16)), 5813 DAG.getConstant(3, MVT::i16)); 5814 5815 5816 return DAG.getNode((VT.getSizeInBits() < 16 ? 5817 ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); 5818} 5819 5820SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) { 5821 MVT VT = Op.getValueType(); 5822 MVT OpVT = VT; 5823 unsigned NumBits = VT.getSizeInBits(); 5824 5825 Op = Op.getOperand(0); 5826 if (VT == MVT::i8) { 5827 // Zero extend to i32 since there is not an i8 bsr. 5828 OpVT = MVT::i32; 5829 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5830 } 5831 5832 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 5833 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5834 Op = DAG.getNode(X86ISD::BSR, VTs, Op); 5835 5836 // If src is zero (i.e. bsr sets ZF), returns NumBits. 5837 SmallVector<SDValue, 4> Ops; 5838 Ops.push_back(Op); 5839 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 5840 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5841 Ops.push_back(Op.getValue(1)); 5842 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5843 5844 // Finally xor with NumBits-1. 5845 Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 5846 5847 if (VT == MVT::i8) 5848 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5849 return Op; 5850} 5851 5852SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) { 5853 MVT VT = Op.getValueType(); 5854 MVT OpVT = VT; 5855 unsigned NumBits = VT.getSizeInBits(); 5856 5857 Op = Op.getOperand(0); 5858 if (VT == MVT::i8) { 5859 OpVT = MVT::i32; 5860 Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); 5861 } 5862 5863 // Issue a bsf (scan bits forward) which also sets EFLAGS. 5864 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 5865 Op = DAG.getNode(X86ISD::BSF, VTs, Op); 5866 5867 // If src is zero (i.e. bsf sets ZF), returns NumBits. 5868 SmallVector<SDValue, 4> Ops; 5869 Ops.push_back(Op); 5870 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 5871 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 5872 Ops.push_back(Op.getValue(1)); 5873 Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); 5874 5875 if (VT == MVT::i8) 5876 Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); 5877 return Op; 5878} 5879 5880SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) { 5881 MVT T = Op.getValueType(); 5882 unsigned Reg = 0; 5883 unsigned size = 0; 5884 switch(T.getSimpleVT()) { 5885 default: 5886 assert(false && "Invalid value type!"); 5887 case MVT::i8: Reg = X86::AL; size = 1; break; 5888 case MVT::i16: Reg = X86::AX; size = 2; break; 5889 case MVT::i32: Reg = X86::EAX; size = 4; break; 5890 case MVT::i64: 5891 if (Subtarget->is64Bit()) { 5892 Reg = X86::RAX; size = 8; 5893 } else //Should go away when LowerType stuff lands 5894 return SDValue(ExpandATOMIC_CMP_SWAP(Op.getNode(), DAG), 0); 5895 break; 5896 }; 5897 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, 5898 Op.getOperand(3), SDValue()); 5899 SDValue Ops[] = { cpIn.getValue(0), 5900 Op.getOperand(1), 5901 Op.getOperand(2), 5902 DAG.getTargetConstant(size, MVT::i8), 5903 cpIn.getValue(1) }; 5904 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5905 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); 5906 SDValue cpOut = 5907 DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); 5908 return cpOut; 5909} 5910 5911SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op, 5912 SelectionDAG &DAG) { 5913 MVT T = Op->getValueType(0); 5914 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 5915 SDValue cpInL, cpInH; 5916 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5917 DAG.getConstant(0, MVT::i32)); 5918 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3), 5919 DAG.getConstant(1, MVT::i32)); 5920 cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX, 5921 cpInL, SDValue()); 5922 cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, 5923 cpInH, cpInL.getValue(1)); 5924 SDValue swapInL, swapInH; 5925 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5926 DAG.getConstant(0, MVT::i32)); 5927 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2), 5928 DAG.getConstant(1, MVT::i32)); 5929 swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, 5930 swapInL, cpInH.getValue(1)); 5931 swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, 5932 swapInH, swapInL.getValue(1)); 5933 SDValue Ops[] = { swapInH.getValue(0), 5934 Op->getOperand(1), 5935 swapInH.getValue(1)}; 5936 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 5937 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); 5938 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 5939 Result.getValue(1)); 5940 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 5941 cpOutL.getValue(2)); 5942 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 5943 SDValue ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2); 5944 SDValue Vals[2] = { ResultVal, cpOutH.getValue(1) }; 5945 return DAG.getMergeValues(Vals, 2).getNode(); 5946} 5947 5948SDNode* X86TargetLowering::ExpandATOMIC_LOAD_SUB(SDNode* Op, 5949 SelectionDAG &DAG) { 5950 MVT T = Op->getValueType(0); 5951 SDValue negOp = DAG.getNode(ISD::SUB, T, 5952 DAG.getConstant(0, T), Op->getOperand(2)); 5953 return DAG.getAtomic((T==MVT::i8 ? ISD::ATOMIC_LOAD_ADD_8: 5954 T==MVT::i16 ? ISD::ATOMIC_LOAD_ADD_16: 5955 T==MVT::i32 ? ISD::ATOMIC_LOAD_ADD_32: 5956 T==MVT::i64 ? ISD::ATOMIC_LOAD_ADD_64: 0), 5957 Op->getOperand(0), Op->getOperand(1), negOp, 5958 cast<AtomicSDNode>(Op)->getSrcValue(), 5959 cast<AtomicSDNode>(Op)->getAlignment()).getNode(); 5960} 5961 5962/// LowerOperation - Provide custom lowering hooks for some operations. 5963/// 5964SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { 5965 switch (Op.getOpcode()) { 5966 default: assert(0 && "Should not custom lower this!"); 5967 case ISD::ATOMIC_CMP_SWAP_8: return LowerCMP_SWAP(Op,DAG); 5968 case ISD::ATOMIC_CMP_SWAP_16: return LowerCMP_SWAP(Op,DAG); 5969 case ISD::ATOMIC_CMP_SWAP_32: return LowerCMP_SWAP(Op,DAG); 5970 case ISD::ATOMIC_CMP_SWAP_64: return LowerCMP_SWAP(Op,DAG); 5971 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 5972 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 5973 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 5974 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 5975 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 5976 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 5977 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 5978 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 5979 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 5980 case ISD::SHL_PARTS: 5981 case ISD::SRA_PARTS: 5982 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 5983 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 5984 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 5985 case ISD::FABS: return LowerFABS(Op, DAG); 5986 case ISD::FNEG: return LowerFNEG(Op, DAG); 5987 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 5988 case ISD::SETCC: return LowerSETCC(Op, DAG); 5989 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 5990 case ISD::SELECT: return LowerSELECT(Op, DAG); 5991 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 5992 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 5993 case ISD::CALL: return LowerCALL(Op, DAG); 5994 case ISD::RET: return LowerRET(Op, DAG); 5995 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); 5996 case ISD::VASTART: return LowerVASTART(Op, DAG); 5997 case ISD::VAARG: return LowerVAARG(Op, DAG); 5998 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 5999 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 6000 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 6001 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 6002 case ISD::FRAME_TO_ARGS_OFFSET: 6003 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 6004 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 6005 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 6006 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 6007 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 6008 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 6009 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 6010 6011 // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. 6012 case ISD::READCYCLECOUNTER: 6013 return SDValue(ExpandREADCYCLECOUNTER(Op.getNode(), DAG), 0); 6014 } 6015} 6016 6017/// ReplaceNodeResults - Replace a node with an illegal result type 6018/// with a new node built out of custom code. 6019SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) { 6020 switch (N->getOpcode()) { 6021 default: assert(0 && "Should not custom lower this!"); 6022 case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); 6023 case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); 6024 case ISD::ATOMIC_CMP_SWAP_64: return ExpandATOMIC_CMP_SWAP(N, DAG); 6025 case ISD::ATOMIC_LOAD_SUB_8: return ExpandATOMIC_LOAD_SUB(N,DAG); 6026 case ISD::ATOMIC_LOAD_SUB_16: return ExpandATOMIC_LOAD_SUB(N,DAG); 6027 case ISD::ATOMIC_LOAD_SUB_32: return ExpandATOMIC_LOAD_SUB(N,DAG); 6028 case ISD::ATOMIC_LOAD_SUB_64: return ExpandATOMIC_LOAD_SUB(N,DAG); 6029 } 6030} 6031 6032const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 6033 switch (Opcode) { 6034 default: return NULL; 6035 case X86ISD::BSF: return "X86ISD::BSF"; 6036 case X86ISD::BSR: return "X86ISD::BSR"; 6037 case X86ISD::SHLD: return "X86ISD::SHLD"; 6038 case X86ISD::SHRD: return "X86ISD::SHRD"; 6039 case X86ISD::FAND: return "X86ISD::FAND"; 6040 case X86ISD::FOR: return "X86ISD::FOR"; 6041 case X86ISD::FXOR: return "X86ISD::FXOR"; 6042 case X86ISD::FSRL: return "X86ISD::FSRL"; 6043 case X86ISD::FILD: return "X86ISD::FILD"; 6044 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 6045 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 6046 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 6047 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 6048 case X86ISD::FLD: return "X86ISD::FLD"; 6049 case X86ISD::FST: return "X86ISD::FST"; 6050 case X86ISD::CALL: return "X86ISD::CALL"; 6051 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 6052 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 6053 case X86ISD::CMP: return "X86ISD::CMP"; 6054 case X86ISD::COMI: return "X86ISD::COMI"; 6055 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 6056 case X86ISD::SETCC: return "X86ISD::SETCC"; 6057 case X86ISD::CMOV: return "X86ISD::CMOV"; 6058 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 6059 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 6060 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 6061 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 6062 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 6063 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 6064 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 6065 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 6066 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 6067 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 6068 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 6069 case X86ISD::FMAX: return "X86ISD::FMAX"; 6070 case X86ISD::FMIN: return "X86ISD::FMIN"; 6071 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 6072 case X86ISD::FRCP: return "X86ISD::FRCP"; 6073 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 6074 case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; 6075 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 6076 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 6077 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 6078 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 6079 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 6080 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 6081 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 6082 case X86ISD::VSHL: return "X86ISD::VSHL"; 6083 case X86ISD::VSRL: return "X86ISD::VSRL"; 6084 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 6085 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 6086 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 6087 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 6088 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 6089 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 6090 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 6091 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 6092 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 6093 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 6094 } 6095} 6096 6097// isLegalAddressingMode - Return true if the addressing mode represented 6098// by AM is legal for this target, for a load/store of the specified type. 6099bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 6100 const Type *Ty) const { 6101 // X86 supports extremely general addressing modes. 6102 6103 // X86 allows a sign-extended 32-bit immediate field as a displacement. 6104 if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) 6105 return false; 6106 6107 if (AM.BaseGV) { 6108 // We can only fold this if we don't need an extra load. 6109 if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) 6110 return false; 6111 6112 // X86-64 only supports addr of globals in small code model. 6113 if (Subtarget->is64Bit()) { 6114 if (getTargetMachine().getCodeModel() != CodeModel::Small) 6115 return false; 6116 // If lower 4G is not available, then we must use rip-relative addressing. 6117 if (AM.BaseOffs || AM.Scale > 1) 6118 return false; 6119 } 6120 } 6121 6122 switch (AM.Scale) { 6123 case 0: 6124 case 1: 6125 case 2: 6126 case 4: 6127 case 8: 6128 // These scales always work. 6129 break; 6130 case 3: 6131 case 5: 6132 case 9: 6133 // These scales are formed with basereg+scalereg. Only accept if there is 6134 // no basereg yet. 6135 if (AM.HasBaseReg) 6136 return false; 6137 break; 6138 default: // Other stuff never works. 6139 return false; 6140 } 6141 6142 return true; 6143} 6144 6145 6146bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 6147 if (!Ty1->isInteger() || !Ty2->isInteger()) 6148 return false; 6149 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6150 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6151 if (NumBits1 <= NumBits2) 6152 return false; 6153 return Subtarget->is64Bit() || NumBits1 < 64; 6154} 6155 6156bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { 6157 if (!VT1.isInteger() || !VT2.isInteger()) 6158 return false; 6159 unsigned NumBits1 = VT1.getSizeInBits(); 6160 unsigned NumBits2 = VT2.getSizeInBits(); 6161 if (NumBits1 <= NumBits2) 6162 return false; 6163 return Subtarget->is64Bit() || NumBits1 < 64; 6164} 6165 6166/// isShuffleMaskLegal - Targets can use this to indicate that they only 6167/// support *some* VECTOR_SHUFFLE operations, those with specific masks. 6168/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 6169/// are assumed to be legal. 6170bool 6171X86TargetLowering::isShuffleMaskLegal(SDValue Mask, MVT VT) const { 6172 // Only do shuffles on 128-bit vector types for now. 6173 if (VT.getSizeInBits() == 64) return false; 6174 return (Mask.getNode()->getNumOperands() <= 4 || 6175 isIdentityMask(Mask.getNode()) || 6176 isIdentityMask(Mask.getNode(), true) || 6177 isSplatMask(Mask.getNode()) || 6178 isPSHUFHW_PSHUFLWMask(Mask.getNode()) || 6179 X86::isUNPCKLMask(Mask.getNode()) || 6180 X86::isUNPCKHMask(Mask.getNode()) || 6181 X86::isUNPCKL_v_undef_Mask(Mask.getNode()) || 6182 X86::isUNPCKH_v_undef_Mask(Mask.getNode())); 6183} 6184 6185bool 6186X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, 6187 MVT EVT, SelectionDAG &DAG) const { 6188 unsigned NumElts = BVOps.size(); 6189 // Only do shuffles on 128-bit vector types for now. 6190 if (EVT.getSizeInBits() * NumElts == 64) return false; 6191 if (NumElts == 2) return true; 6192 if (NumElts == 4) { 6193 return (isMOVLMask(&BVOps[0], 4) || 6194 isCommutedMOVL(&BVOps[0], 4, true) || 6195 isSHUFPMask(&BVOps[0], 4) || 6196 isCommutedSHUFP(&BVOps[0], 4)); 6197 } 6198 return false; 6199} 6200 6201//===----------------------------------------------------------------------===// 6202// X86 Scheduler Hooks 6203//===----------------------------------------------------------------------===// 6204 6205// private utility function 6206MachineBasicBlock * 6207X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 6208 MachineBasicBlock *MBB, 6209 unsigned regOpc, 6210 unsigned immOpc, 6211 unsigned LoadOpc, 6212 unsigned CXchgOpc, 6213 unsigned copyOpc, 6214 unsigned notOpc, 6215 unsigned EAXreg, 6216 TargetRegisterClass *RC, 6217 bool invSrc) { 6218 // For the atomic bitwise operator, we generate 6219 // thisMBB: 6220 // newMBB: 6221 // ld t1 = [bitinstr.addr] 6222 // op t2 = t1, [bitinstr.val] 6223 // mov EAX = t1 6224 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6225 // bz newMBB 6226 // fallthrough -->nextMBB 6227 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6228 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6229 MachineFunction::iterator MBBIter = MBB; 6230 ++MBBIter; 6231 6232 /// First build the CFG 6233 MachineFunction *F = MBB->getParent(); 6234 MachineBasicBlock *thisMBB = MBB; 6235 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6236 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6237 F->insert(MBBIter, newMBB); 6238 F->insert(MBBIter, nextMBB); 6239 6240 // Move all successors to thisMBB to nextMBB 6241 nextMBB->transferSuccessors(thisMBB); 6242 6243 // Update thisMBB to fall through to newMBB 6244 thisMBB->addSuccessor(newMBB); 6245 6246 // newMBB jumps to itself and fall through to nextMBB 6247 newMBB->addSuccessor(nextMBB); 6248 newMBB->addSuccessor(newMBB); 6249 6250 // Insert instructions into newMBB based on incoming instruction 6251 assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); 6252 MachineOperand& destOper = bInstr->getOperand(0); 6253 MachineOperand* argOpers[6]; 6254 int numArgs = bInstr->getNumOperands() - 1; 6255 for (int i=0; i < numArgs; ++i) 6256 argOpers[i] = &bInstr->getOperand(i+1); 6257 6258 // x86 address has 4 operands: base, index, scale, and displacement 6259 int lastAddrIndx = 3; // [0,3] 6260 int valArgIndx = 4; 6261 6262 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 6263 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(LoadOpc), t1); 6264 for (int i=0; i <= lastAddrIndx; ++i) 6265 (*MIB).addOperand(*argOpers[i]); 6266 6267 unsigned tt = F->getRegInfo().createVirtualRegister(RC); 6268 if (invSrc) { 6269 MIB = BuildMI(newMBB, TII->get(notOpc), tt).addReg(t1); 6270 } 6271 else 6272 tt = t1; 6273 6274 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 6275 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6276 && "invalid operand"); 6277 if (argOpers[valArgIndx]->isReg()) 6278 MIB = BuildMI(newMBB, TII->get(regOpc), t2); 6279 else 6280 MIB = BuildMI(newMBB, TII->get(immOpc), t2); 6281 MIB.addReg(tt); 6282 (*MIB).addOperand(*argOpers[valArgIndx]); 6283 6284 MIB = BuildMI(newMBB, TII->get(copyOpc), EAXreg); 6285 MIB.addReg(t1); 6286 6287 MIB = BuildMI(newMBB, TII->get(CXchgOpc)); 6288 for (int i=0; i <= lastAddrIndx; ++i) 6289 (*MIB).addOperand(*argOpers[i]); 6290 MIB.addReg(t2); 6291 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6292 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 6293 6294 MIB = BuildMI(newMBB, TII->get(copyOpc), destOper.getReg()); 6295 MIB.addReg(EAXreg); 6296 6297 // insert branch 6298 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6299 6300 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 6301 return nextMBB; 6302} 6303 6304// private utility function 6305MachineBasicBlock * 6306X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 6307 MachineBasicBlock *MBB, 6308 unsigned cmovOpc) { 6309 // For the atomic min/max operator, we generate 6310 // thisMBB: 6311 // newMBB: 6312 // ld t1 = [min/max.addr] 6313 // mov t2 = [min/max.val] 6314 // cmp t1, t2 6315 // cmov[cond] t2 = t1 6316 // mov EAX = t1 6317 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 6318 // bz newMBB 6319 // fallthrough -->nextMBB 6320 // 6321 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6322 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 6323 MachineFunction::iterator MBBIter = MBB; 6324 ++MBBIter; 6325 6326 /// First build the CFG 6327 MachineFunction *F = MBB->getParent(); 6328 MachineBasicBlock *thisMBB = MBB; 6329 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 6330 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 6331 F->insert(MBBIter, newMBB); 6332 F->insert(MBBIter, nextMBB); 6333 6334 // Move all successors to thisMBB to nextMBB 6335 nextMBB->transferSuccessors(thisMBB); 6336 6337 // Update thisMBB to fall through to newMBB 6338 thisMBB->addSuccessor(newMBB); 6339 6340 // newMBB jumps to newMBB and fall through to nextMBB 6341 newMBB->addSuccessor(nextMBB); 6342 newMBB->addSuccessor(newMBB); 6343 6344 // Insert instructions into newMBB based on incoming instruction 6345 assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); 6346 MachineOperand& destOper = mInstr->getOperand(0); 6347 MachineOperand* argOpers[6]; 6348 int numArgs = mInstr->getNumOperands() - 1; 6349 for (int i=0; i < numArgs; ++i) 6350 argOpers[i] = &mInstr->getOperand(i+1); 6351 6352 // x86 address has 4 operands: base, index, scale, and displacement 6353 int lastAddrIndx = 3; // [0,3] 6354 int valArgIndx = 4; 6355 6356 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6357 MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); 6358 for (int i=0; i <= lastAddrIndx; ++i) 6359 (*MIB).addOperand(*argOpers[i]); 6360 6361 // We only support register and immediate values 6362 assert( (argOpers[valArgIndx]->isReg() || argOpers[valArgIndx]->isImm()) 6363 && "invalid operand"); 6364 6365 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6366 if (argOpers[valArgIndx]->isReg()) 6367 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6368 else 6369 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); 6370 (*MIB).addOperand(*argOpers[valArgIndx]); 6371 6372 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); 6373 MIB.addReg(t1); 6374 6375 MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); 6376 MIB.addReg(t1); 6377 MIB.addReg(t2); 6378 6379 // Generate movc 6380 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 6381 MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); 6382 MIB.addReg(t2); 6383 MIB.addReg(t1); 6384 6385 // Cmp and exchange if none has modified the memory location 6386 MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); 6387 for (int i=0; i <= lastAddrIndx; ++i) 6388 (*MIB).addOperand(*argOpers[i]); 6389 MIB.addReg(t3); 6390 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 6391 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 6392 6393 MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); 6394 MIB.addReg(X86::EAX); 6395 6396 // insert branch 6397 BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); 6398 6399 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 6400 return nextMBB; 6401} 6402 6403 6404MachineBasicBlock * 6405X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6406 MachineBasicBlock *BB) { 6407 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6408 switch (MI->getOpcode()) { 6409 default: assert(false && "Unexpected instr type to insert"); 6410 case X86::CMOV_FR32: 6411 case X86::CMOV_FR64: 6412 case X86::CMOV_V4F32: 6413 case X86::CMOV_V2F64: 6414 case X86::CMOV_V2I64: { 6415 // To "insert" a SELECT_CC instruction, we actually have to insert the 6416 // diamond control-flow pattern. The incoming instruction knows the 6417 // destination vreg to set, the condition code register to branch on, the 6418 // true/false values to select between, and a branch opcode to use. 6419 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6420 MachineFunction::iterator It = BB; 6421 ++It; 6422 6423 // thisMBB: 6424 // ... 6425 // TrueVal = ... 6426 // cmpTY ccX, r1, r2 6427 // bCC copy1MBB 6428 // fallthrough --> copy0MBB 6429 MachineBasicBlock *thisMBB = BB; 6430 MachineFunction *F = BB->getParent(); 6431 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6432 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6433 unsigned Opc = 6434 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 6435 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); 6436 F->insert(It, copy0MBB); 6437 F->insert(It, sinkMBB); 6438 // Update machine-CFG edges by transferring all successors of the current 6439 // block to the new block which will contain the Phi node for the select. 6440 sinkMBB->transferSuccessors(BB); 6441 6442 // Add the true and fallthrough blocks as its successors. 6443 BB->addSuccessor(copy0MBB); 6444 BB->addSuccessor(sinkMBB); 6445 6446 // copy0MBB: 6447 // %FalseValue = ... 6448 // # fallthrough to sinkMBB 6449 BB = copy0MBB; 6450 6451 // Update machine-CFG edges 6452 BB->addSuccessor(sinkMBB); 6453 6454 // sinkMBB: 6455 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6456 // ... 6457 BB = sinkMBB; 6458 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) 6459 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 6460 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6461 6462 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6463 return BB; 6464 } 6465 6466 case X86::FP32_TO_INT16_IN_MEM: 6467 case X86::FP32_TO_INT32_IN_MEM: 6468 case X86::FP32_TO_INT64_IN_MEM: 6469 case X86::FP64_TO_INT16_IN_MEM: 6470 case X86::FP64_TO_INT32_IN_MEM: 6471 case X86::FP64_TO_INT64_IN_MEM: 6472 case X86::FP80_TO_INT16_IN_MEM: 6473 case X86::FP80_TO_INT32_IN_MEM: 6474 case X86::FP80_TO_INT64_IN_MEM: { 6475 // Change the floating point control register to use "round towards zero" 6476 // mode when truncating to an integer value. 6477 MachineFunction *F = BB->getParent(); 6478 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 6479 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 6480 6481 // Load the old value of the high byte of the control word... 6482 unsigned OldCW = 6483 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 6484 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); 6485 6486 // Set the high part to be round to zero... 6487 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) 6488 .addImm(0xC7F); 6489 6490 // Reload the modified control word now... 6491 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6492 6493 // Restore the memory image of control word to original value 6494 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) 6495 .addReg(OldCW); 6496 6497 // Get the X86 opcode to use. 6498 unsigned Opc; 6499 switch (MI->getOpcode()) { 6500 default: assert(0 && "illegal opcode!"); 6501 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 6502 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 6503 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 6504 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 6505 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 6506 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 6507 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 6508 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 6509 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 6510 } 6511 6512 X86AddressMode AM; 6513 MachineOperand &Op = MI->getOperand(0); 6514 if (Op.isRegister()) { 6515 AM.BaseType = X86AddressMode::RegBase; 6516 AM.Base.Reg = Op.getReg(); 6517 } else { 6518 AM.BaseType = X86AddressMode::FrameIndexBase; 6519 AM.Base.FrameIndex = Op.getIndex(); 6520 } 6521 Op = MI->getOperand(1); 6522 if (Op.isImmediate()) 6523 AM.Scale = Op.getImm(); 6524 Op = MI->getOperand(2); 6525 if (Op.isImmediate()) 6526 AM.IndexReg = Op.getImm(); 6527 Op = MI->getOperand(3); 6528 if (Op.isGlobalAddress()) { 6529 AM.GV = Op.getGlobal(); 6530 } else { 6531 AM.Disp = Op.getImm(); 6532 } 6533 addFullAddress(BuildMI(BB, TII->get(Opc)), AM) 6534 .addReg(MI->getOperand(4).getReg()); 6535 6536 // Reload the original control word now. 6537 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 6538 6539 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 6540 return BB; 6541 } 6542 case X86::ATOMAND32: 6543 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6544 X86::AND32ri, X86::MOV32rm, 6545 X86::LCMPXCHG32, X86::MOV32rr, 6546 X86::NOT32r, X86::EAX, 6547 X86::GR32RegisterClass); 6548 case X86::ATOMOR32: 6549 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 6550 X86::OR32ri, X86::MOV32rm, 6551 X86::LCMPXCHG32, X86::MOV32rr, 6552 X86::NOT32r, X86::EAX, 6553 X86::GR32RegisterClass); 6554 case X86::ATOMXOR32: 6555 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 6556 X86::XOR32ri, X86::MOV32rm, 6557 X86::LCMPXCHG32, X86::MOV32rr, 6558 X86::NOT32r, X86::EAX, 6559 X86::GR32RegisterClass); 6560 case X86::ATOMNAND32: 6561 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 6562 X86::AND32ri, X86::MOV32rm, 6563 X86::LCMPXCHG32, X86::MOV32rr, 6564 X86::NOT32r, X86::EAX, 6565 X86::GR32RegisterClass, true); 6566 case X86::ATOMMIN32: 6567 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 6568 case X86::ATOMMAX32: 6569 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 6570 case X86::ATOMUMIN32: 6571 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 6572 case X86::ATOMUMAX32: 6573 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 6574 6575 case X86::ATOMAND16: 6576 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6577 X86::AND16ri, X86::MOV16rm, 6578 X86::LCMPXCHG16, X86::MOV16rr, 6579 X86::NOT16r, X86::AX, 6580 X86::GR16RegisterClass); 6581 case X86::ATOMOR16: 6582 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 6583 X86::OR16ri, X86::MOV16rm, 6584 X86::LCMPXCHG16, X86::MOV16rr, 6585 X86::NOT16r, X86::AX, 6586 X86::GR16RegisterClass); 6587 case X86::ATOMXOR16: 6588 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 6589 X86::XOR16ri, X86::MOV16rm, 6590 X86::LCMPXCHG16, X86::MOV16rr, 6591 X86::NOT16r, X86::AX, 6592 X86::GR16RegisterClass); 6593 case X86::ATOMNAND16: 6594 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 6595 X86::AND16ri, X86::MOV16rm, 6596 X86::LCMPXCHG16, X86::MOV16rr, 6597 X86::NOT16r, X86::AX, 6598 X86::GR16RegisterClass, true); 6599 case X86::ATOMMIN16: 6600 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 6601 case X86::ATOMMAX16: 6602 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 6603 case X86::ATOMUMIN16: 6604 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 6605 case X86::ATOMUMAX16: 6606 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 6607 6608 case X86::ATOMAND8: 6609 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6610 X86::AND8ri, X86::MOV8rm, 6611 X86::LCMPXCHG8, X86::MOV8rr, 6612 X86::NOT8r, X86::AL, 6613 X86::GR8RegisterClass); 6614 case X86::ATOMOR8: 6615 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 6616 X86::OR8ri, X86::MOV8rm, 6617 X86::LCMPXCHG8, X86::MOV8rr, 6618 X86::NOT8r, X86::AL, 6619 X86::GR8RegisterClass); 6620 case X86::ATOMXOR8: 6621 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 6622 X86::XOR8ri, X86::MOV8rm, 6623 X86::LCMPXCHG8, X86::MOV8rr, 6624 X86::NOT8r, X86::AL, 6625 X86::GR8RegisterClass); 6626 case X86::ATOMNAND8: 6627 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 6628 X86::AND8ri, X86::MOV8rm, 6629 X86::LCMPXCHG8, X86::MOV8rr, 6630 X86::NOT8r, X86::AL, 6631 X86::GR8RegisterClass, true); 6632 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 6633 case X86::ATOMAND64: 6634 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 6635 X86::AND64ri32, X86::MOV64rm, 6636 X86::LCMPXCHG64, X86::MOV64rr, 6637 X86::NOT64r, X86::RAX, 6638 X86::GR64RegisterClass); 6639 case X86::ATOMOR64: 6640 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, 6641 X86::OR64ri32, X86::MOV64rm, 6642 X86::LCMPXCHG64, X86::MOV64rr, 6643 X86::NOT64r, X86::RAX, 6644 X86::GR64RegisterClass); 6645 case X86::ATOMXOR64: 6646 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, 6647 X86::XOR64ri32, X86::MOV64rm, 6648 X86::LCMPXCHG64, X86::MOV64rr, 6649 X86::NOT64r, X86::RAX, 6650 X86::GR64RegisterClass); 6651 case X86::ATOMNAND64: 6652 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 6653 X86::AND64ri32, X86::MOV64rm, 6654 X86::LCMPXCHG64, X86::MOV64rr, 6655 X86::NOT64r, X86::RAX, 6656 X86::GR64RegisterClass, true); 6657 case X86::ATOMMIN64: 6658 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); 6659 case X86::ATOMMAX64: 6660 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); 6661 case X86::ATOMUMIN64: 6662 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); 6663 case X86::ATOMUMAX64: 6664 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); 6665 } 6666} 6667 6668//===----------------------------------------------------------------------===// 6669// X86 Optimization Hooks 6670//===----------------------------------------------------------------------===// 6671 6672void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 6673 const APInt &Mask, 6674 APInt &KnownZero, 6675 APInt &KnownOne, 6676 const SelectionDAG &DAG, 6677 unsigned Depth) const { 6678 unsigned Opc = Op.getOpcode(); 6679 assert((Opc >= ISD::BUILTIN_OP_END || 6680 Opc == ISD::INTRINSIC_WO_CHAIN || 6681 Opc == ISD::INTRINSIC_W_CHAIN || 6682 Opc == ISD::INTRINSIC_VOID) && 6683 "Should use MaskedValueIsZero if you don't know whether Op" 6684 " is a target node!"); 6685 6686 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 6687 switch (Opc) { 6688 default: break; 6689 case X86ISD::SETCC: 6690 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 6691 Mask.getBitWidth() - 1); 6692 break; 6693 } 6694} 6695 6696/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 6697/// node is a GlobalAddress + offset. 6698bool X86TargetLowering::isGAPlusOffset(SDNode *N, 6699 GlobalValue* &GA, int64_t &Offset) const{ 6700 if (N->getOpcode() == X86ISD::Wrapper) { 6701 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 6702 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 6703 return true; 6704 } 6705 } 6706 return TargetLowering::isGAPlusOffset(N, GA, Offset); 6707} 6708 6709static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 6710 const TargetLowering &TLI) { 6711 GlobalValue *GV; 6712 int64_t Offset = 0; 6713 if (TLI.isGAPlusOffset(Base, GV, Offset)) 6714 return (GV->getAlignment() >= N && (Offset % N) == 0); 6715 // DAG combine handles the stack object case. 6716 return false; 6717} 6718 6719static bool EltsFromConsecutiveLoads(SDNode *N, SDValue PermMask, 6720 unsigned NumElems, MVT EVT, 6721 SDNode *&Base, 6722 SelectionDAG &DAG, MachineFrameInfo *MFI, 6723 const TargetLowering &TLI) { 6724 Base = NULL; 6725 for (unsigned i = 0; i < NumElems; ++i) { 6726 SDValue Idx = PermMask.getOperand(i); 6727 if (Idx.getOpcode() == ISD::UNDEF) { 6728 if (!Base) 6729 return false; 6730 continue; 6731 } 6732 6733 SDValue Elt = DAG.getShuffleScalarElt(N, i); 6734 if (!Elt.getNode() || 6735 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) 6736 return false; 6737 if (!Base) { 6738 Base = Elt.getNode(); 6739 if (Base->getOpcode() == ISD::UNDEF) 6740 return false; 6741 continue; 6742 } 6743 if (Elt.getOpcode() == ISD::UNDEF) 6744 continue; 6745 6746 if (!TLI.isConsecutiveLoad(Elt.getNode(), Base, 6747 EVT.getSizeInBits()/8, i, MFI)) 6748 return false; 6749 } 6750 return true; 6751} 6752 6753/// PerformShuffleCombine - Combine a vector_shuffle that is equal to 6754/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 6755/// if the load addresses are consecutive, non-overlapping, and in the right 6756/// order. 6757static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 6758 const TargetLowering &TLI) { 6759 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 6760 MVT VT = N->getValueType(0); 6761 MVT EVT = VT.getVectorElementType(); 6762 SDValue PermMask = N->getOperand(2); 6763 unsigned NumElems = PermMask.getNumOperands(); 6764 SDNode *Base = NULL; 6765 if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, 6766 DAG, MFI, TLI)) 6767 return SDValue(); 6768 6769 LoadSDNode *LD = cast<LoadSDNode>(Base); 6770 if (isBaseAlignmentOfN(16, Base->getOperand(1).getNode(), TLI)) 6771 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6772 LD->getSrcValueOffset(), LD->isVolatile()); 6773 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), 6774 LD->getSrcValueOffset(), LD->isVolatile(), 6775 LD->getAlignment()); 6776} 6777 6778/// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. 6779static SDValue PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, 6780 const X86Subtarget *Subtarget, 6781 const TargetLowering &TLI) { 6782 unsigned NumOps = N->getNumOperands(); 6783 6784 // Ignore single operand BUILD_VECTOR. 6785 if (NumOps == 1) 6786 return SDValue(); 6787 6788 MVT VT = N->getValueType(0); 6789 MVT EVT = VT.getVectorElementType(); 6790 if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) 6791 // We are looking for load i64 and zero extend. We want to transform 6792 // it before legalizer has a chance to expand it. Also look for i64 6793 // BUILD_PAIR bit casted to f64. 6794 return SDValue(); 6795 // This must be an insertion into a zero vector. 6796 SDValue HighElt = N->getOperand(1); 6797 if (!isZeroNode(HighElt)) 6798 return SDValue(); 6799 6800 // Value must be a load. 6801 SDNode *Base = N->getOperand(0).getNode(); 6802 if (!isa<LoadSDNode>(Base)) { 6803 if (Base->getOpcode() != ISD::BIT_CONVERT) 6804 return SDValue(); 6805 Base = Base->getOperand(0).getNode(); 6806 if (!isa<LoadSDNode>(Base)) 6807 return SDValue(); 6808 } 6809 6810 // Transform it into VZEXT_LOAD addr. 6811 LoadSDNode *LD = cast<LoadSDNode>(Base); 6812 6813 // Load must not be an extload. 6814 if (LD->getExtensionType() != ISD::NON_EXTLOAD) 6815 return SDValue(); 6816 6817 return DAG.getNode(X86ISD::VZEXT_LOAD, VT, LD->getChain(), LD->getBasePtr()); 6818} 6819 6820/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 6821static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 6822 const X86Subtarget *Subtarget) { 6823 SDValue Cond = N->getOperand(0); 6824 6825 // If we have SSE[12] support, try to form min/max nodes. 6826 if (Subtarget->hasSSE2() && 6827 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { 6828 if (Cond.getOpcode() == ISD::SETCC) { 6829 // Get the LHS/RHS of the select. 6830 SDValue LHS = N->getOperand(1); 6831 SDValue RHS = N->getOperand(2); 6832 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 6833 6834 unsigned Opcode = 0; 6835 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 6836 switch (CC) { 6837 default: break; 6838 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 6839 case ISD::SETULE: 6840 case ISD::SETLE: 6841 if (!UnsafeFPMath) break; 6842 // FALL THROUGH. 6843 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 6844 case ISD::SETLT: 6845 Opcode = X86ISD::FMIN; 6846 break; 6847 6848 case ISD::SETOGT: // (X > Y) ? X : Y -> max 6849 case ISD::SETUGT: 6850 case ISD::SETGT: 6851 if (!UnsafeFPMath) break; 6852 // FALL THROUGH. 6853 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 6854 case ISD::SETGE: 6855 Opcode = X86ISD::FMAX; 6856 break; 6857 } 6858 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 6859 switch (CC) { 6860 default: break; 6861 case ISD::SETOGT: // (X > Y) ? Y : X -> min 6862 case ISD::SETUGT: 6863 case ISD::SETGT: 6864 if (!UnsafeFPMath) break; 6865 // FALL THROUGH. 6866 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 6867 case ISD::SETGE: 6868 Opcode = X86ISD::FMIN; 6869 break; 6870 6871 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 6872 case ISD::SETULE: 6873 case ISD::SETLE: 6874 if (!UnsafeFPMath) break; 6875 // FALL THROUGH. 6876 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 6877 case ISD::SETLT: 6878 Opcode = X86ISD::FMAX; 6879 break; 6880 } 6881 } 6882 6883 if (Opcode) 6884 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); 6885 } 6886 6887 } 6888 6889 return SDValue(); 6890} 6891 6892/// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 6893static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 6894 const X86Subtarget *Subtarget) { 6895 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 6896 // the FP state in cases where an emms may be missing. 6897 // A preferable solution to the general problem is to figure out the right 6898 // places to insert EMMS. This qualifies as a quick hack. 6899 StoreSDNode *St = cast<StoreSDNode>(N); 6900 if (St->getValue().getValueType().isVector() && 6901 St->getValue().getValueType().getSizeInBits() == 64 && 6902 isa<LoadSDNode>(St->getValue()) && 6903 !cast<LoadSDNode>(St->getValue())->isVolatile() && 6904 St->getChain().hasOneUse() && !St->isVolatile()) { 6905 SDNode* LdVal = St->getValue().getNode(); 6906 LoadSDNode *Ld = 0; 6907 int TokenFactorIndex = -1; 6908 SmallVector<SDValue, 8> Ops; 6909 SDNode* ChainVal = St->getChain().getNode(); 6910 // Must be a store of a load. We currently handle two cases: the load 6911 // is a direct child, and it's under an intervening TokenFactor. It is 6912 // possible to dig deeper under nested TokenFactors. 6913 if (ChainVal == LdVal) 6914 Ld = cast<LoadSDNode>(St->getChain()); 6915 else if (St->getValue().hasOneUse() && 6916 ChainVal->getOpcode() == ISD::TokenFactor) { 6917 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 6918 if (ChainVal->getOperand(i).getNode() == LdVal) { 6919 TokenFactorIndex = i; 6920 Ld = cast<LoadSDNode>(St->getValue()); 6921 } else 6922 Ops.push_back(ChainVal->getOperand(i)); 6923 } 6924 } 6925 if (Ld) { 6926 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 6927 if (Subtarget->is64Bit()) { 6928 SDValue NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), 6929 Ld->getBasePtr(), Ld->getSrcValue(), 6930 Ld->getSrcValueOffset(), Ld->isVolatile(), 6931 Ld->getAlignment()); 6932 SDValue NewChain = NewLd.getValue(1); 6933 if (TokenFactorIndex != -1) { 6934 Ops.push_back(NewChain); 6935 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6936 Ops.size()); 6937 } 6938 return DAG.getStore(NewChain, NewLd, St->getBasePtr(), 6939 St->getSrcValue(), St->getSrcValueOffset(), 6940 St->isVolatile(), St->getAlignment()); 6941 } 6942 6943 // Otherwise, lower to two 32-bit copies. 6944 SDValue LoAddr = Ld->getBasePtr(); 6945 SDValue HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6946 DAG.getConstant(4, MVT::i32)); 6947 6948 SDValue LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, 6949 Ld->getSrcValue(), Ld->getSrcValueOffset(), 6950 Ld->isVolatile(), Ld->getAlignment()); 6951 SDValue HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, 6952 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 6953 Ld->isVolatile(), 6954 MinAlign(Ld->getAlignment(), 4)); 6955 6956 SDValue NewChain = LoLd.getValue(1); 6957 if (TokenFactorIndex != -1) { 6958 Ops.push_back(LoLd); 6959 Ops.push_back(HiLd); 6960 NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], 6961 Ops.size()); 6962 } 6963 6964 LoAddr = St->getBasePtr(); 6965 HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, 6966 DAG.getConstant(4, MVT::i32)); 6967 6968 SDValue LoSt = DAG.getStore(NewChain, LoLd, LoAddr, 6969 St->getSrcValue(), St->getSrcValueOffset(), 6970 St->isVolatile(), St->getAlignment()); 6971 SDValue HiSt = DAG.getStore(NewChain, HiLd, HiAddr, 6972 St->getSrcValue(), 6973 St->getSrcValueOffset() + 4, 6974 St->isVolatile(), 6975 MinAlign(St->getAlignment(), 4)); 6976 return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); 6977 } 6978 } 6979 return SDValue(); 6980} 6981 6982/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 6983/// X86ISD::FXOR nodes. 6984static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 6985 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 6986 // F[X]OR(0.0, x) -> x 6987 // F[X]OR(x, 0.0) -> x 6988 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 6989 if (C->getValueAPF().isPosZero()) 6990 return N->getOperand(1); 6991 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 6992 if (C->getValueAPF().isPosZero()) 6993 return N->getOperand(0); 6994 return SDValue(); 6995} 6996 6997/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 6998static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 6999 // FAND(0.0, x) -> 0.0 7000 // FAND(x, 0.0) -> 0.0 7001 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 7002 if (C->getValueAPF().isPosZero()) 7003 return N->getOperand(0); 7004 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 7005 if (C->getValueAPF().isPosZero()) 7006 return N->getOperand(1); 7007 return SDValue(); 7008} 7009 7010 7011SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 7012 DAGCombinerInfo &DCI) const { 7013 SelectionDAG &DAG = DCI.DAG; 7014 switch (N->getOpcode()) { 7015 default: break; 7016 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 7017 case ISD::BUILD_VECTOR: 7018 return PerformBuildVectorCombine(N, DAG, Subtarget, *this); 7019 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 7020 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 7021 case X86ISD::FXOR: 7022 case X86ISD::FOR: return PerformFORCombine(N, DAG); 7023 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 7024 } 7025 7026 return SDValue(); 7027} 7028 7029//===----------------------------------------------------------------------===// 7030// X86 Inline Assembly Support 7031//===----------------------------------------------------------------------===// 7032 7033/// getConstraintType - Given a constraint letter, return the type of 7034/// constraint it is for this target. 7035X86TargetLowering::ConstraintType 7036X86TargetLowering::getConstraintType(const std::string &Constraint) const { 7037 if (Constraint.size() == 1) { 7038 switch (Constraint[0]) { 7039 case 'A': 7040 case 'f': 7041 case 'r': 7042 case 'R': 7043 case 'l': 7044 case 'q': 7045 case 'Q': 7046 case 'x': 7047 case 'y': 7048 case 'Y': 7049 return C_RegisterClass; 7050 default: 7051 break; 7052 } 7053 } 7054 return TargetLowering::getConstraintType(Constraint); 7055} 7056 7057/// LowerXConstraint - try to replace an X constraint, which matches anything, 7058/// with another that has more specific requirements based on the type of the 7059/// corresponding operand. 7060const char *X86TargetLowering:: 7061LowerXConstraint(MVT ConstraintVT) const { 7062 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 7063 // 'f' like normal targets. 7064 if (ConstraintVT.isFloatingPoint()) { 7065 if (Subtarget->hasSSE2()) 7066 return "Y"; 7067 if (Subtarget->hasSSE1()) 7068 return "x"; 7069 } 7070 7071 return TargetLowering::LowerXConstraint(ConstraintVT); 7072} 7073 7074/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 7075/// vector. If it is invalid, don't add anything to Ops. 7076void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 7077 char Constraint, 7078 std::vector<SDValue>&Ops, 7079 SelectionDAG &DAG) const { 7080 SDValue Result(0, 0); 7081 7082 switch (Constraint) { 7083 default: break; 7084 case 'I': 7085 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 7086 if (C->getValue() <= 31) { 7087 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 7088 break; 7089 } 7090 } 7091 return; 7092 case 'N': 7093 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 7094 if (C->getValue() <= 255) { 7095 Result = DAG.getTargetConstant(C->getValue(), Op.getValueType()); 7096 break; 7097 } 7098 } 7099 return; 7100 case 'i': { 7101 // Literal immediates are always ok. 7102 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 7103 Result = DAG.getTargetConstant(CST->getValue(), Op.getValueType()); 7104 break; 7105 } 7106 7107 // If we are in non-pic codegen mode, we allow the address of a global (with 7108 // an optional displacement) to be used with 'i'. 7109 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); 7110 int64_t Offset = 0; 7111 7112 // Match either (GA) or (GA+C) 7113 if (GA) { 7114 Offset = GA->getOffset(); 7115 } else if (Op.getOpcode() == ISD::ADD) { 7116 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7117 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7118 if (C && GA) { 7119 Offset = GA->getOffset()+C->getValue(); 7120 } else { 7121 C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 7122 GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); 7123 if (C && GA) 7124 Offset = GA->getOffset()+C->getValue(); 7125 else 7126 C = 0, GA = 0; 7127 } 7128 } 7129 7130 if (GA) { 7131 // If addressing this global requires a load (e.g. in PIC mode), we can't 7132 // match. 7133 if (Subtarget->GVRequiresExtraLoad(GA->getGlobal(), getTargetMachine(), 7134 false)) 7135 return; 7136 7137 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 7138 Offset); 7139 Result = Op; 7140 break; 7141 } 7142 7143 // Otherwise, not valid for this mode. 7144 return; 7145 } 7146 } 7147 7148 if (Result.getNode()) { 7149 Ops.push_back(Result); 7150 return; 7151 } 7152 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 7153} 7154 7155std::vector<unsigned> X86TargetLowering:: 7156getRegClassForInlineAsmConstraint(const std::string &Constraint, 7157 MVT VT) const { 7158 if (Constraint.size() == 1) { 7159 // FIXME: not handling fp-stack yet! 7160 switch (Constraint[0]) { // GCC X86 Constraint Letters 7161 default: break; // Unknown constraint letter 7162 case 'A': // EAX/EDX 7163 if (VT == MVT::i32 || VT == MVT::i64) 7164 return make_vector<unsigned>(X86::EAX, X86::EDX, 0); 7165 break; 7166 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) 7167 case 'Q': // Q_REGS 7168 if (VT == MVT::i32) 7169 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 7170 else if (VT == MVT::i16) 7171 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 7172 else if (VT == MVT::i8) 7173 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 7174 else if (VT == MVT::i64) 7175 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 7176 break; 7177 } 7178 } 7179 7180 return std::vector<unsigned>(); 7181} 7182 7183std::pair<unsigned, const TargetRegisterClass*> 7184X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 7185 MVT VT) const { 7186 // First, see if this is a constraint that directly corresponds to an LLVM 7187 // register class. 7188 if (Constraint.size() == 1) { 7189 // GCC Constraint Letters 7190 switch (Constraint[0]) { 7191 default: break; 7192 case 'r': // GENERAL_REGS 7193 case 'R': // LEGACY_REGS 7194 case 'l': // INDEX_REGS 7195 if (VT == MVT::i64 && Subtarget->is64Bit()) 7196 return std::make_pair(0U, X86::GR64RegisterClass); 7197 if (VT == MVT::i32) 7198 return std::make_pair(0U, X86::GR32RegisterClass); 7199 else if (VT == MVT::i16) 7200 return std::make_pair(0U, X86::GR16RegisterClass); 7201 else if (VT == MVT::i8) 7202 return std::make_pair(0U, X86::GR8RegisterClass); 7203 break; 7204 case 'f': // FP Stack registers. 7205 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 7206 // value to the correct fpstack register class. 7207 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 7208 return std::make_pair(0U, X86::RFP32RegisterClass); 7209 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 7210 return std::make_pair(0U, X86::RFP64RegisterClass); 7211 return std::make_pair(0U, X86::RFP80RegisterClass); 7212 case 'y': // MMX_REGS if MMX allowed. 7213 if (!Subtarget->hasMMX()) break; 7214 return std::make_pair(0U, X86::VR64RegisterClass); 7215 break; 7216 case 'Y': // SSE_REGS if SSE2 allowed 7217 if (!Subtarget->hasSSE2()) break; 7218 // FALL THROUGH. 7219 case 'x': // SSE_REGS if SSE1 allowed 7220 if (!Subtarget->hasSSE1()) break; 7221 7222 switch (VT.getSimpleVT()) { 7223 default: break; 7224 // Scalar SSE types. 7225 case MVT::f32: 7226 case MVT::i32: 7227 return std::make_pair(0U, X86::FR32RegisterClass); 7228 case MVT::f64: 7229 case MVT::i64: 7230 return std::make_pair(0U, X86::FR64RegisterClass); 7231 // Vector types. 7232 case MVT::v16i8: 7233 case MVT::v8i16: 7234 case MVT::v4i32: 7235 case MVT::v2i64: 7236 case MVT::v4f32: 7237 case MVT::v2f64: 7238 return std::make_pair(0U, X86::VR128RegisterClass); 7239 } 7240 break; 7241 } 7242 } 7243 7244 // Use the default implementation in TargetLowering to convert the register 7245 // constraint into a member of a register class. 7246 std::pair<unsigned, const TargetRegisterClass*> Res; 7247 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 7248 7249 // Not found as a standard register? 7250 if (Res.second == 0) { 7251 // GCC calls "st(0)" just plain "st". 7252 if (StringsEqualNoCase("{st}", Constraint)) { 7253 Res.first = X86::ST0; 7254 Res.second = X86::RFP80RegisterClass; 7255 } 7256 7257 return Res; 7258 } 7259 7260 // Otherwise, check to see if this is a register class of the wrong value 7261 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 7262 // turn into {ax},{dx}. 7263 if (Res.second->hasType(VT)) 7264 return Res; // Correct type already, nothing to do. 7265 7266 // All of the single-register GCC register classes map their values onto 7267 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 7268 // really want an 8-bit or 32-bit register, map to the appropriate register 7269 // class and return the appropriate register. 7270 if (Res.second == X86::GR16RegisterClass) { 7271 if (VT == MVT::i8) { 7272 unsigned DestReg = 0; 7273 switch (Res.first) { 7274 default: break; 7275 case X86::AX: DestReg = X86::AL; break; 7276 case X86::DX: DestReg = X86::DL; break; 7277 case X86::CX: DestReg = X86::CL; break; 7278 case X86::BX: DestReg = X86::BL; break; 7279 } 7280 if (DestReg) { 7281 Res.first = DestReg; 7282 Res.second = Res.second = X86::GR8RegisterClass; 7283 } 7284 } else if (VT == MVT::i32) { 7285 unsigned DestReg = 0; 7286 switch (Res.first) { 7287 default: break; 7288 case X86::AX: DestReg = X86::EAX; break; 7289 case X86::DX: DestReg = X86::EDX; break; 7290 case X86::CX: DestReg = X86::ECX; break; 7291 case X86::BX: DestReg = X86::EBX; break; 7292 case X86::SI: DestReg = X86::ESI; break; 7293 case X86::DI: DestReg = X86::EDI; break; 7294 case X86::BP: DestReg = X86::EBP; break; 7295 case X86::SP: DestReg = X86::ESP; break; 7296 } 7297 if (DestReg) { 7298 Res.first = DestReg; 7299 Res.second = Res.second = X86::GR32RegisterClass; 7300 } 7301 } else if (VT == MVT::i64) { 7302 unsigned DestReg = 0; 7303 switch (Res.first) { 7304 default: break; 7305 case X86::AX: DestReg = X86::RAX; break; 7306 case X86::DX: DestReg = X86::RDX; break; 7307 case X86::CX: DestReg = X86::RCX; break; 7308 case X86::BX: DestReg = X86::RBX; break; 7309 case X86::SI: DestReg = X86::RSI; break; 7310 case X86::DI: DestReg = X86::RDI; break; 7311 case X86::BP: DestReg = X86::RBP; break; 7312 case X86::SP: DestReg = X86::RSP; break; 7313 } 7314 if (DestReg) { 7315 Res.first = DestReg; 7316 Res.second = Res.second = X86::GR64RegisterClass; 7317 } 7318 } 7319 } else if (Res.second == X86::FR32RegisterClass || 7320 Res.second == X86::FR64RegisterClass || 7321 Res.second == X86::VR128RegisterClass) { 7322 // Handle references to XMM physical registers that got mapped into the 7323 // wrong class. This can happen with constraints like {xmm0} where the 7324 // target independent register mapper will just pick the first match it can 7325 // find, ignoring the required type. 7326 if (VT == MVT::f32) 7327 Res.second = X86::FR32RegisterClass; 7328 else if (VT == MVT::f64) 7329 Res.second = X86::FR64RegisterClass; 7330 else if (X86::VR128RegisterClass->hasType(VT)) 7331 Res.second = X86::VR128RegisterClass; 7332 } 7333 7334 return Res; 7335} 7336