X86ISelLowering.cpp revision 357c58efdeb4eef9bf220795ba89a873709486c9
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by Chris Lattner and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86TargetMachine.h" 19#include "llvm/CallingConv.h" 20#include "llvm/Function.h" 21#include "llvm/CodeGen/MachineFrameInfo.h" 22#include "llvm/CodeGen/MachineFunction.h" 23#include "llvm/CodeGen/MachineInstrBuilder.h" 24#include "llvm/CodeGen/SelectionDAG.h" 25#include "llvm/CodeGen/SSARegMap.h" 26#include "llvm/Target/TargetOptions.h" 27using namespace llvm; 28 29// FIXME: temporary. 30#include "llvm/Support/CommandLine.h" 31static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden, 32 cl::desc("Enable fastcc on X86")); 33 34X86TargetLowering::X86TargetLowering(TargetMachine &TM) 35 : TargetLowering(TM) { 36 // Set up the TargetLowering object. 37 38 // X86 is weird, it always uses i8 for shift amounts and setcc results. 39 setShiftAmountType(MVT::i8); 40 setSetCCResultType(MVT::i8); 41 setSetCCResultContents(ZeroOrOneSetCCResult); 42 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 43 setStackPointerRegisterToSaveRestore(X86::ESP); 44 45 // Set up the register classes. 46 addRegisterClass(MVT::i8, X86::R8RegisterClass); 47 addRegisterClass(MVT::i16, X86::R16RegisterClass); 48 addRegisterClass(MVT::i32, X86::R32RegisterClass); 49 50 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 51 // operation. 52 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 53 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 54 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 55 56 if (X86ScalarSSE) 57 // No SSE i64 SINT_TO_FP, so expand i32 UINT_TO_FP instead. 58 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 59 else 60 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 61 62 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 63 // this operation. 64 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 65 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 66 67 if (!X86ScalarSSE) { 68 // We can handle SINT_TO_FP and FP_TO_SINT from/TO i64 even though i64 69 // isn't legal. 70 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 71 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 72 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 73 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 74 } 75 76 // Handle FP_TO_UINT by promoting the destination to a larger signed 77 // conversion. 78 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 79 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 80 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 81 82 if (!X86ScalarSSE) 83 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 84 85 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 86 // this operation. 87 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 88 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 89 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 90 91 setOperationAction(ISD::BIT_CONVERT, MVT::f32, Expand); 92 setOperationAction(ISD::BIT_CONVERT, MVT::i32, Expand); 93 94 if (X86DAGIsel) { 95 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 96 } 97 setOperationAction(ISD::BRCONDTWOWAY , MVT::Other, Expand); 98 setOperationAction(ISD::BRTWOWAY_CC , MVT::Other, Expand); 99 setOperationAction(ISD::MEMMOVE , MVT::Other, Expand); 100 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand); 101 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand); 102 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 103 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 104 setOperationAction(ISD::SEXTLOAD , MVT::i1 , Expand); 105 setOperationAction(ISD::FREM , MVT::f64 , Expand); 106 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 107 setOperationAction(ISD::CTTZ , MVT::i8 , Expand); 108 setOperationAction(ISD::CTLZ , MVT::i8 , Expand); 109 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 110 setOperationAction(ISD::CTTZ , MVT::i16 , Expand); 111 setOperationAction(ISD::CTLZ , MVT::i16 , Expand); 112 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 113 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 114 setOperationAction(ISD::CTLZ , MVT::i32 , Expand); 115 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 116 117 if (!X86DAGIsel) { 118 setOperationAction(ISD::BSWAP , MVT::i32 , Expand); 119 setOperationAction(ISD::ROTL , MVT::i8 , Expand); 120 setOperationAction(ISD::ROTR , MVT::i8 , Expand); 121 setOperationAction(ISD::ROTL , MVT::i16 , Expand); 122 setOperationAction(ISD::ROTR , MVT::i16 , Expand); 123 setOperationAction(ISD::ROTL , MVT::i32 , Expand); 124 setOperationAction(ISD::ROTR , MVT::i32 , Expand); 125 } 126 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 127 128 setOperationAction(ISD::READIO , MVT::i1 , Expand); 129 setOperationAction(ISD::READIO , MVT::i8 , Expand); 130 setOperationAction(ISD::READIO , MVT::i16 , Expand); 131 setOperationAction(ISD::READIO , MVT::i32 , Expand); 132 setOperationAction(ISD::WRITEIO , MVT::i1 , Expand); 133 setOperationAction(ISD::WRITEIO , MVT::i8 , Expand); 134 setOperationAction(ISD::WRITEIO , MVT::i16 , Expand); 135 setOperationAction(ISD::WRITEIO , MVT::i32 , Expand); 136 137 // These should be promoted to a larger select which is supported. 138 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 139 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 140 if (X86DAGIsel) { 141 // X86 wants to expand cmov itself. 142 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 143 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 144 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 145 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 146 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 147 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 148 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 149 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 150 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 151 // X86 ret instruction may pop stack. 152 setOperationAction(ISD::RET , MVT::Other, Custom); 153 // Darwin ABI issue. 154 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 155 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 156 setOperationAction(ISD::ADD_PARTS , MVT::i32 , Custom); 157 setOperationAction(ISD::SUB_PARTS , MVT::i32 , Custom); 158 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 159 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 160 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 161 // X86 wants to expand memset / memcpy itself. 162 setOperationAction(ISD::MEMSET , MVT::Other, Custom); 163 setOperationAction(ISD::MEMCPY , MVT::Other, Custom); 164 } 165 166 // We don't have line number support yet. 167 setOperationAction(ISD::LOCATION, MVT::Other, Expand); 168 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand); 169 setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand); 170 171 // Expand to the default code. 172 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 173 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 174 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand); 175 176 if (X86ScalarSSE) { 177 // Set up the FP register classes. 178 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 179 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 180 181 // SSE has no load+extend ops 182 setOperationAction(ISD::EXTLOAD, MVT::f32, Expand); 183 setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand); 184 185 // SSE has no i16 to fp conversion, only i32 186 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); 187 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); 188 189 // Expand FP_TO_UINT into a select. 190 // FIXME: We would like to use a Custom expander here eventually to do 191 // the optimal thing for SSE vs. the default expansion in the legalizer. 192 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 193 194 // We don't support sin/cos/sqrt/fmod 195 setOperationAction(ISD::FSIN , MVT::f64, Expand); 196 setOperationAction(ISD::FCOS , MVT::f64, Expand); 197 setOperationAction(ISD::FABS , MVT::f64, Expand); 198 setOperationAction(ISD::FNEG , MVT::f64, Expand); 199 setOperationAction(ISD::FREM , MVT::f64, Expand); 200 setOperationAction(ISD::FSIN , MVT::f32, Expand); 201 setOperationAction(ISD::FCOS , MVT::f32, Expand); 202 setOperationAction(ISD::FABS , MVT::f32, Expand); 203 setOperationAction(ISD::FNEG , MVT::f32, Expand); 204 setOperationAction(ISD::FREM , MVT::f32, Expand); 205 206 addLegalFPImmediate(+0.0); // xorps / xorpd 207 } else { 208 // Set up the FP register classes. 209 addRegisterClass(MVT::f64, X86::RFPRegisterClass); 210 211 if (X86DAGIsel) { 212 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); 213 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 214 } 215 216 if (!UnsafeFPMath) { 217 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 218 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 219 } 220 221 addLegalFPImmediate(+0.0); // FLD0 222 addLegalFPImmediate(+1.0); // FLD1 223 addLegalFPImmediate(-0.0); // FLD0/FCHS 224 addLegalFPImmediate(-1.0); // FLD1/FCHS 225 } 226 computeRegisterProperties(); 227 228 maxStoresPerMemSet = 8; // For %llvm.memset -> sequence of stores 229 maxStoresPerMemCpy = 8; // For %llvm.memcpy -> sequence of stores 230 maxStoresPerMemMove = 8; // For %llvm.memmove -> sequence of stores 231 allowUnalignedMemoryAccesses = true; // x86 supports it! 232} 233 234std::vector<SDOperand> 235X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { 236 if (F.getCallingConv() == CallingConv::Fast && EnableFastCC) 237 return LowerFastCCArguments(F, DAG); 238 return LowerCCCArguments(F, DAG); 239} 240 241std::pair<SDOperand, SDOperand> 242X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, 243 bool isVarArg, unsigned CallingConv, 244 bool isTailCall, 245 SDOperand Callee, ArgListTy &Args, 246 SelectionDAG &DAG) { 247 assert((!isVarArg || CallingConv == CallingConv::C) && 248 "Only C takes varargs!"); 249 250 // If the callee is a GlobalAddress node (quite common, every direct call is) 251 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 252 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 253 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 254 else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) 255 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 256 257 if (CallingConv == CallingConv::Fast && EnableFastCC) 258 return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG); 259 return LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG); 260} 261 262SDOperand X86TargetLowering::LowerReturnTo(SDOperand Chain, SDOperand Op, 263 SelectionDAG &DAG) { 264 if (!X86DAGIsel) 265 return DAG.getNode(ISD::RET, MVT::Other, Chain, Op); 266 267 SDOperand Copy; 268 MVT::ValueType OpVT = Op.getValueType(); 269 switch (OpVT) { 270 default: assert(0 && "Unknown type to return!"); 271 case MVT::i32: 272 Copy = DAG.getCopyToReg(Chain, X86::EAX, Op, SDOperand()); 273 break; 274 case MVT::i64: { 275 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op, 276 DAG.getConstant(1, MVT::i32)); 277 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op, 278 DAG.getConstant(0, MVT::i32)); 279 Copy = DAG.getCopyToReg(Chain, X86::EDX, Hi, SDOperand()); 280 Copy = DAG.getCopyToReg(Copy, X86::EAX, Lo, Copy.getValue(1)); 281 break; 282 } 283 case MVT::f32: 284 case MVT::f64: 285 if (!X86ScalarSSE) { 286 if (OpVT == MVT::f32) 287 Op = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Op); 288 std::vector<MVT::ValueType> Tys; 289 Tys.push_back(MVT::Other); 290 Tys.push_back(MVT::Flag); 291 std::vector<SDOperand> Ops; 292 Ops.push_back(Chain); 293 Ops.push_back(Op); 294 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops); 295 } else { 296 // Spill the value to memory and reload it into top of stack. 297 unsigned Size = MVT::getSizeInBits(OpVT)/8; 298 MachineFunction &MF = DAG.getMachineFunction(); 299 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 300 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 301 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Op, 302 StackSlot, DAG.getSrcValue(NULL)); 303 std::vector<MVT::ValueType> Tys; 304 Tys.push_back(MVT::f64); 305 Tys.push_back(MVT::Other); 306 std::vector<SDOperand> Ops; 307 Ops.push_back(Chain); 308 Ops.push_back(StackSlot); 309 Ops.push_back(DAG.getValueType(OpVT)); 310 Copy = DAG.getNode(X86ISD::FLD, Tys, Ops); 311 Tys.clear(); 312 Tys.push_back(MVT::Other); 313 Tys.push_back(MVT::Flag); 314 Ops.clear(); 315 Ops.push_back(Copy.getValue(1)); 316 Ops.push_back(Copy); 317 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops); 318 } 319 break; 320 } 321 322 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, 323 Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16), 324 Copy.getValue(1)); 325} 326 327//===----------------------------------------------------------------------===// 328// C Calling Convention implementation 329//===----------------------------------------------------------------------===// 330 331std::vector<SDOperand> 332X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) { 333 std::vector<SDOperand> ArgValues; 334 335 MachineFunction &MF = DAG.getMachineFunction(); 336 MachineFrameInfo *MFI = MF.getFrameInfo(); 337 338 // Add DAG nodes to load the arguments... On entry to a function on the X86, 339 // the stack frame looks like this: 340 // 341 // [ESP] -- return address 342 // [ESP + 4] -- first argument (leftmost lexically) 343 // [ESP + 8] -- second argument, if first argument is four bytes in size 344 // ... 345 // 346 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot 347 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) { 348 MVT::ValueType ObjectVT = getValueType(I->getType()); 349 unsigned ArgIncrement = 4; 350 unsigned ObjSize; 351 switch (ObjectVT) { 352 default: assert(0 && "Unhandled argument type!"); 353 case MVT::i1: 354 case MVT::i8: ObjSize = 1; break; 355 case MVT::i16: ObjSize = 2; break; 356 case MVT::i32: ObjSize = 4; break; 357 case MVT::i64: ObjSize = ArgIncrement = 8; break; 358 case MVT::f32: ObjSize = 4; break; 359 case MVT::f64: ObjSize = ArgIncrement = 8; break; 360 } 361 // Create the frame index object for this incoming parameter... 362 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); 363 364 // Create the SelectionDAG nodes corresponding to a load from this parameter 365 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 366 367 // Don't codegen dead arguments. FIXME: remove this check when we can nuke 368 // dead loads. 369 SDOperand ArgValue; 370 if (!I->use_empty()) 371 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN, 372 DAG.getSrcValue(NULL)); 373 else { 374 if (MVT::isInteger(ObjectVT)) 375 ArgValue = DAG.getConstant(0, ObjectVT); 376 else 377 ArgValue = DAG.getConstantFP(0, ObjectVT); 378 } 379 ArgValues.push_back(ArgValue); 380 381 ArgOffset += ArgIncrement; // Move on to the next argument... 382 } 383 384 // If the function takes variable number of arguments, make a frame index for 385 // the start of the first vararg value... for expansion of llvm.va_start. 386 if (F.isVarArg()) 387 VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset); 388 ReturnAddrIndex = 0; // No return address slot generated yet. 389 BytesToPopOnReturn = 0; // Callee pops nothing. 390 BytesCallerReserves = ArgOffset; 391 392 // Finally, inform the code generator which regs we return values in. 393 switch (getValueType(F.getReturnType())) { 394 default: assert(0 && "Unknown type!"); 395 case MVT::isVoid: break; 396 case MVT::i1: 397 case MVT::i8: 398 case MVT::i16: 399 case MVT::i32: 400 MF.addLiveOut(X86::EAX); 401 break; 402 case MVT::i64: 403 MF.addLiveOut(X86::EAX); 404 MF.addLiveOut(X86::EDX); 405 break; 406 case MVT::f32: 407 case MVT::f64: 408 MF.addLiveOut(X86::ST0); 409 break; 410 } 411 return ArgValues; 412} 413 414std::pair<SDOperand, SDOperand> 415X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy, 416 bool isVarArg, bool isTailCall, 417 SDOperand Callee, ArgListTy &Args, 418 SelectionDAG &DAG) { 419 // Count how many bytes are to be pushed on the stack. 420 unsigned NumBytes = 0; 421 422 if (Args.empty()) { 423 // Save zero bytes. 424 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 425 DAG.getConstant(0, getPointerTy())); 426 } else { 427 for (unsigned i = 0, e = Args.size(); i != e; ++i) 428 switch (getValueType(Args[i].second)) { 429 default: assert(0 && "Unknown value type!"); 430 case MVT::i1: 431 case MVT::i8: 432 case MVT::i16: 433 case MVT::i32: 434 case MVT::f32: 435 NumBytes += 4; 436 break; 437 case MVT::i64: 438 case MVT::f64: 439 NumBytes += 8; 440 break; 441 } 442 443 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 444 DAG.getConstant(NumBytes, getPointerTy())); 445 446 // Arguments go on the stack in reverse order, as specified by the ABI. 447 unsigned ArgOffset = 0; 448 SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32); 449 std::vector<SDOperand> Stores; 450 451 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 452 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 453 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 454 455 switch (getValueType(Args[i].second)) { 456 default: assert(0 && "Unexpected ValueType for argument!"); 457 case MVT::i1: 458 case MVT::i8: 459 case MVT::i16: 460 // Promote the integer to 32 bits. If the input type is signed use a 461 // sign extend, otherwise use a zero extend. 462 if (Args[i].second->isSigned()) 463 Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first); 464 else 465 Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first); 466 467 // FALL THROUGH 468 case MVT::i32: 469 case MVT::f32: 470 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 471 Args[i].first, PtrOff, 472 DAG.getSrcValue(NULL))); 473 ArgOffset += 4; 474 break; 475 case MVT::i64: 476 case MVT::f64: 477 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 478 Args[i].first, PtrOff, 479 DAG.getSrcValue(NULL))); 480 ArgOffset += 8; 481 break; 482 } 483 } 484 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); 485 } 486 487 std::vector<MVT::ValueType> RetVals; 488 MVT::ValueType RetTyVT = getValueType(RetTy); 489 RetVals.push_back(MVT::Other); 490 491 // The result values produced have to be legal. Promote the result. 492 switch (RetTyVT) { 493 case MVT::isVoid: break; 494 default: 495 RetVals.push_back(RetTyVT); 496 break; 497 case MVT::i1: 498 case MVT::i8: 499 case MVT::i16: 500 RetVals.push_back(MVT::i32); 501 break; 502 case MVT::f32: 503 if (X86ScalarSSE) 504 RetVals.push_back(MVT::f32); 505 else 506 RetVals.push_back(MVT::f64); 507 break; 508 case MVT::i64: 509 RetVals.push_back(MVT::i32); 510 RetVals.push_back(MVT::i32); 511 break; 512 } 513 514 if (X86DAGIsel) { 515 std::vector<MVT::ValueType> NodeTys; 516 NodeTys.push_back(MVT::Other); // Returns a chain 517 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 518 std::vector<SDOperand> Ops; 519 Ops.push_back(Chain); 520 Ops.push_back(Callee); 521 522 // FIXME: Do not generate X86ISD::TAILCALL for now. 523 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops); 524 SDOperand InFlag = Chain.getValue(1); 525 526 SDOperand RetVal; 527 if (RetTyVT != MVT::isVoid) { 528 switch (RetTyVT) { 529 default: assert(0 && "Unknown value type to return!"); 530 case MVT::i1: 531 case MVT::i8: 532 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag); 533 Chain = RetVal.getValue(1); 534 break; 535 case MVT::i16: 536 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); 537 Chain = RetVal.getValue(1); 538 break; 539 case MVT::i32: 540 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 541 Chain = RetVal.getValue(1); 542 break; 543 case MVT::i64: { 544 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 545 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32, 546 Lo.getValue(2)); 547 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); 548 Chain = Hi.getValue(1); 549 break; 550 } 551 case MVT::f32: 552 case MVT::f64: { 553 std::vector<MVT::ValueType> Tys; 554 Tys.push_back(MVT::f64); 555 Tys.push_back(MVT::Other); 556 Tys.push_back(MVT::Flag); 557 std::vector<SDOperand> Ops; 558 Ops.push_back(Chain); 559 Ops.push_back(InFlag); 560 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops); 561 Chain = RetVal.getValue(1); 562 InFlag = RetVal.getValue(2); 563 if (X86ScalarSSE) { 564 // FIXME:Currently the FST is flagged to the FP_GET_RESULT. This 565 // shouldn't be necessary except for RFP cannot be live across 566 // multiple blocks. When stackifier is fixed, they can be uncoupled. 567 unsigned Size = MVT::getSizeInBits(MVT::f64)/8; 568 MachineFunction &MF = DAG.getMachineFunction(); 569 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 570 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 571 Tys.clear(); 572 Tys.push_back(MVT::Other); 573 Ops.clear(); 574 Ops.push_back(Chain); 575 Ops.push_back(RetVal); 576 Ops.push_back(StackSlot); 577 Ops.push_back(DAG.getValueType(RetTyVT)); 578 Ops.push_back(InFlag); 579 Chain = DAG.getNode(X86ISD::FST, Tys, Ops); 580 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot, 581 DAG.getSrcValue(NULL)); 582 Chain = RetVal.getValue(1); 583 } 584 585 if (RetTyVT == MVT::f32 && !X86ScalarSSE) 586 // FIXME: we would really like to remember that this FP_ROUND 587 // operation is okay to eliminate if we allow excess FP precision. 588 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); 589 break; 590 } 591 } 592 } 593 594 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain, 595 DAG.getConstant(NumBytes, getPointerTy()), 596 DAG.getConstant(0, getPointerTy())); 597 return std::make_pair(RetVal, Chain); 598 } else { 599 std::vector<SDOperand> Ops; 600 Ops.push_back(Chain); 601 Ops.push_back(Callee); 602 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); 603 Ops.push_back(DAG.getConstant(0, getPointerTy())); 604 605 SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, 606 RetVals, Ops); 607 608 SDOperand ResultVal; 609 switch (RetTyVT) { 610 case MVT::isVoid: break; 611 default: 612 ResultVal = TheCall.getValue(1); 613 break; 614 case MVT::i1: 615 case MVT::i8: 616 case MVT::i16: 617 ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1)); 618 break; 619 case MVT::f32: 620 // FIXME: we would really like to remember that this FP_ROUND operation is 621 // okay to eliminate if we allow excess FP precision. 622 ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1)); 623 break; 624 case MVT::i64: 625 ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1), 626 TheCall.getValue(2)); 627 break; 628 } 629 630 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall); 631 return std::make_pair(ResultVal, Chain); 632 } 633} 634 635SDOperand 636X86TargetLowering::LowerVAStart(SDOperand Chain, SDOperand VAListP, 637 Value *VAListV, SelectionDAG &DAG) { 638 // vastart just stores the address of the VarArgsFrameIndex slot. 639 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32); 640 return DAG.getNode(ISD::STORE, MVT::Other, Chain, FR, VAListP, 641 DAG.getSrcValue(VAListV)); 642} 643 644 645std::pair<SDOperand,SDOperand> 646X86TargetLowering::LowerVAArg(SDOperand Chain, SDOperand VAListP, 647 Value *VAListV, const Type *ArgTy, 648 SelectionDAG &DAG) { 649 MVT::ValueType ArgVT = getValueType(ArgTy); 650 SDOperand Val = DAG.getLoad(MVT::i32, Chain, 651 VAListP, DAG.getSrcValue(VAListV)); 652 SDOperand Result = DAG.getLoad(ArgVT, Chain, Val, 653 DAG.getSrcValue(NULL)); 654 unsigned Amt; 655 if (ArgVT == MVT::i32) 656 Amt = 4; 657 else { 658 assert((ArgVT == MVT::i64 || ArgVT == MVT::f64) && 659 "Other types should have been promoted for varargs!"); 660 Amt = 8; 661 } 662 Val = DAG.getNode(ISD::ADD, Val.getValueType(), Val, 663 DAG.getConstant(Amt, Val.getValueType())); 664 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, 665 Val, VAListP, DAG.getSrcValue(VAListV)); 666 return std::make_pair(Result, Chain); 667} 668 669//===----------------------------------------------------------------------===// 670// Fast Calling Convention implementation 671//===----------------------------------------------------------------------===// 672// 673// The X86 'fast' calling convention passes up to two integer arguments in 674// registers (an appropriate portion of EAX/EDX), passes arguments in C order, 675// and requires that the callee pop its arguments off the stack (allowing proper 676// tail calls), and has the same return value conventions as C calling convs. 677// 678// This calling convention always arranges for the callee pop value to be 8n+4 679// bytes, which is needed for tail recursion elimination and stack alignment 680// reasons. 681// 682// Note that this can be enhanced in the future to pass fp vals in registers 683// (when we have a global fp allocator) and do other tricks. 684// 685 686/// AddLiveIn - This helper function adds the specified physical register to the 687/// MachineFunction as a live in value. It also creates a corresponding virtual 688/// register for it. 689static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 690 TargetRegisterClass *RC) { 691 assert(RC->contains(PReg) && "Not the correct regclass!"); 692 unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC); 693 MF.addLiveIn(PReg, VReg); 694 return VReg; 695} 696 697 698std::vector<SDOperand> 699X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) { 700 std::vector<SDOperand> ArgValues; 701 702 MachineFunction &MF = DAG.getMachineFunction(); 703 MachineFrameInfo *MFI = MF.getFrameInfo(); 704 705 // Add DAG nodes to load the arguments... On entry to a function the stack 706 // frame looks like this: 707 // 708 // [ESP] -- return address 709 // [ESP + 4] -- first nonreg argument (leftmost lexically) 710 // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size 711 // ... 712 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot 713 714 // Keep track of the number of integer regs passed so far. This can be either 715 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both 716 // used). 717 unsigned NumIntRegs = 0; 718 719 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) { 720 MVT::ValueType ObjectVT = getValueType(I->getType()); 721 unsigned ArgIncrement = 4; 722 unsigned ObjSize = 0; 723 SDOperand ArgValue; 724 725 switch (ObjectVT) { 726 default: assert(0 && "Unhandled argument type!"); 727 case MVT::i1: 728 case MVT::i8: 729 if (NumIntRegs < 2) { 730 if (!I->use_empty()) { 731 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL, 732 X86::R8RegisterClass); 733 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8); 734 DAG.setRoot(ArgValue.getValue(1)); 735 if (ObjectVT == MVT::i1) 736 // FIXME: Should insert a assertzext here. 737 ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue); 738 } 739 ++NumIntRegs; 740 break; 741 } 742 743 ObjSize = 1; 744 break; 745 case MVT::i16: 746 if (NumIntRegs < 2) { 747 if (!I->use_empty()) { 748 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX, 749 X86::R16RegisterClass); 750 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16); 751 DAG.setRoot(ArgValue.getValue(1)); 752 } 753 ++NumIntRegs; 754 break; 755 } 756 ObjSize = 2; 757 break; 758 case MVT::i32: 759 if (NumIntRegs < 2) { 760 if (!I->use_empty()) { 761 unsigned VReg = AddLiveIn(MF,NumIntRegs ? X86::EDX : X86::EAX, 762 X86::R32RegisterClass); 763 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32); 764 DAG.setRoot(ArgValue.getValue(1)); 765 } 766 ++NumIntRegs; 767 break; 768 } 769 ObjSize = 4; 770 break; 771 case MVT::i64: 772 if (NumIntRegs == 0) { 773 if (!I->use_empty()) { 774 unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass); 775 unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass); 776 777 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32); 778 SDOperand Hi = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32); 779 DAG.setRoot(Hi.getValue(1)); 780 781 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi); 782 } 783 NumIntRegs = 2; 784 break; 785 } else if (NumIntRegs == 1) { 786 if (!I->use_empty()) { 787 unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass); 788 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32); 789 DAG.setRoot(Low.getValue(1)); 790 791 // Load the high part from memory. 792 // Create the frame index object for this incoming parameter... 793 int FI = MFI->CreateFixedObject(4, ArgOffset); 794 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 795 SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN, 796 DAG.getSrcValue(NULL)); 797 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi); 798 } 799 ArgOffset += 4; 800 NumIntRegs = 2; 801 break; 802 } 803 ObjSize = ArgIncrement = 8; 804 break; 805 case MVT::f32: ObjSize = 4; break; 806 case MVT::f64: ObjSize = ArgIncrement = 8; break; 807 } 808 809 // Don't codegen dead arguments. FIXME: remove this check when we can nuke 810 // dead loads. 811 if (ObjSize && !I->use_empty()) { 812 // Create the frame index object for this incoming parameter... 813 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); 814 815 // Create the SelectionDAG nodes corresponding to a load from this 816 // parameter. 817 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 818 819 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN, 820 DAG.getSrcValue(NULL)); 821 } else if (ArgValue.Val == 0) { 822 if (MVT::isInteger(ObjectVT)) 823 ArgValue = DAG.getConstant(0, ObjectVT); 824 else 825 ArgValue = DAG.getConstantFP(0, ObjectVT); 826 } 827 ArgValues.push_back(ArgValue); 828 829 if (ObjSize) 830 ArgOffset += ArgIncrement; // Move on to the next argument. 831 } 832 833 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 834 // arguments and the arguments after the retaddr has been pushed are aligned. 835 if ((ArgOffset & 7) == 0) 836 ArgOffset += 4; 837 838 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 839 ReturnAddrIndex = 0; // No return address slot generated yet. 840 BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments. 841 BytesCallerReserves = 0; 842 843 // Finally, inform the code generator which regs we return values in. 844 switch (getValueType(F.getReturnType())) { 845 default: assert(0 && "Unknown type!"); 846 case MVT::isVoid: break; 847 case MVT::i1: 848 case MVT::i8: 849 case MVT::i16: 850 case MVT::i32: 851 MF.addLiveOut(X86::EAX); 852 break; 853 case MVT::i64: 854 MF.addLiveOut(X86::EAX); 855 MF.addLiveOut(X86::EDX); 856 break; 857 case MVT::f32: 858 case MVT::f64: 859 MF.addLiveOut(X86::ST0); 860 break; 861 } 862 return ArgValues; 863} 864 865std::pair<SDOperand, SDOperand> 866X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy, 867 bool isTailCall, SDOperand Callee, 868 ArgListTy &Args, SelectionDAG &DAG) { 869 // Count how many bytes are to be pushed on the stack. 870 unsigned NumBytes = 0; 871 872 // Keep track of the number of integer regs passed so far. This can be either 873 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both 874 // used). 875 unsigned NumIntRegs = 0; 876 877 for (unsigned i = 0, e = Args.size(); i != e; ++i) 878 switch (getValueType(Args[i].second)) { 879 default: assert(0 && "Unknown value type!"); 880 case MVT::i1: 881 case MVT::i8: 882 case MVT::i16: 883 case MVT::i32: 884 if (NumIntRegs < 2) { 885 ++NumIntRegs; 886 break; 887 } 888 // fall through 889 case MVT::f32: 890 NumBytes += 4; 891 break; 892 case MVT::i64: 893 if (NumIntRegs == 0) { 894 NumIntRegs = 2; 895 break; 896 } else if (NumIntRegs == 1) { 897 NumIntRegs = 2; 898 NumBytes += 4; 899 break; 900 } 901 902 // fall through 903 case MVT::f64: 904 NumBytes += 8; 905 break; 906 } 907 908 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 909 // arguments and the arguments after the retaddr has been pushed are aligned. 910 if ((NumBytes & 7) == 0) 911 NumBytes += 4; 912 913 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 914 DAG.getConstant(NumBytes, getPointerTy())); 915 916 // Arguments go on the stack in reverse order, as specified by the ABI. 917 unsigned ArgOffset = 0; 918 SDOperand StackPtr = DAG.getCopyFromReg(DAG.getEntryNode(), 919 X86::ESP, MVT::i32); 920 NumIntRegs = 0; 921 std::vector<SDOperand> Stores; 922 std::vector<SDOperand> RegValuesToPass; 923 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 924 switch (getValueType(Args[i].second)) { 925 default: assert(0 && "Unexpected ValueType for argument!"); 926 case MVT::i1: 927 Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first); 928 // Fall through. 929 case MVT::i8: 930 case MVT::i16: 931 case MVT::i32: 932 if (NumIntRegs < 2) { 933 RegValuesToPass.push_back(Args[i].first); 934 ++NumIntRegs; 935 break; 936 } 937 // Fall through 938 case MVT::f32: { 939 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 940 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 941 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 942 Args[i].first, PtrOff, 943 DAG.getSrcValue(NULL))); 944 ArgOffset += 4; 945 break; 946 } 947 case MVT::i64: 948 if (NumIntRegs < 2) { // Can pass part of it in regs? 949 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, 950 Args[i].first, DAG.getConstant(1, MVT::i32)); 951 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, 952 Args[i].first, DAG.getConstant(0, MVT::i32)); 953 RegValuesToPass.push_back(Lo); 954 ++NumIntRegs; 955 if (NumIntRegs < 2) { // Pass both parts in regs? 956 RegValuesToPass.push_back(Hi); 957 ++NumIntRegs; 958 } else { 959 // Pass the high part in memory. 960 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 961 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 962 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 963 Hi, PtrOff, DAG.getSrcValue(NULL))); 964 ArgOffset += 4; 965 } 966 break; 967 } 968 // Fall through 969 case MVT::f64: 970 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 971 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 972 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 973 Args[i].first, PtrOff, 974 DAG.getSrcValue(NULL))); 975 ArgOffset += 8; 976 break; 977 } 978 } 979 if (!Stores.empty()) 980 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); 981 982 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 983 // arguments and the arguments after the retaddr has been pushed are aligned. 984 if ((ArgOffset & 7) == 0) 985 ArgOffset += 4; 986 987 std::vector<MVT::ValueType> RetVals; 988 MVT::ValueType RetTyVT = getValueType(RetTy); 989 990 RetVals.push_back(MVT::Other); 991 992 // The result values produced have to be legal. Promote the result. 993 switch (RetTyVT) { 994 case MVT::isVoid: break; 995 default: 996 RetVals.push_back(RetTyVT); 997 break; 998 case MVT::i1: 999 case MVT::i8: 1000 case MVT::i16: 1001 RetVals.push_back(MVT::i32); 1002 break; 1003 case MVT::f32: 1004 if (X86ScalarSSE) 1005 RetVals.push_back(MVT::f32); 1006 else 1007 RetVals.push_back(MVT::f64); 1008 break; 1009 case MVT::i64: 1010 RetVals.push_back(MVT::i32); 1011 RetVals.push_back(MVT::i32); 1012 break; 1013 } 1014 1015 if (X86DAGIsel) { 1016 // Build a sequence of copy-to-reg nodes chained together with token chain 1017 // and flag operands which copy the outgoing args into registers. 1018 SDOperand InFlag; 1019 for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) { 1020 unsigned CCReg; 1021 SDOperand RegToPass = RegValuesToPass[i]; 1022 switch (RegToPass.getValueType()) { 1023 default: assert(0 && "Bad thing to pass in regs"); 1024 case MVT::i8: 1025 CCReg = (i == 0) ? X86::AL : X86::DL; 1026 break; 1027 case MVT::i16: 1028 CCReg = (i == 0) ? X86::AX : X86::DX; 1029 break; 1030 case MVT::i32: 1031 CCReg = (i == 0) ? X86::EAX : X86::EDX; 1032 break; 1033 } 1034 1035 Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag); 1036 InFlag = Chain.getValue(1); 1037 } 1038 1039 std::vector<MVT::ValueType> NodeTys; 1040 NodeTys.push_back(MVT::Other); // Returns a chain 1041 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 1042 std::vector<SDOperand> Ops; 1043 Ops.push_back(Chain); 1044 Ops.push_back(Callee); 1045 if (InFlag.Val) 1046 Ops.push_back(InFlag); 1047 1048 // FIXME: Do not generate X86ISD::TAILCALL for now. 1049 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops); 1050 InFlag = Chain.getValue(1); 1051 1052 SDOperand RetVal; 1053 if (RetTyVT != MVT::isVoid) { 1054 switch (RetTyVT) { 1055 default: assert(0 && "Unknown value type to return!"); 1056 case MVT::i1: 1057 case MVT::i8: 1058 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag); 1059 Chain = RetVal.getValue(1); 1060 break; 1061 case MVT::i16: 1062 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); 1063 Chain = RetVal.getValue(1); 1064 break; 1065 case MVT::i32: 1066 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 1067 Chain = RetVal.getValue(1); 1068 break; 1069 case MVT::i64: { 1070 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 1071 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32, 1072 Lo.getValue(2)); 1073 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); 1074 Chain = Hi.getValue(1); 1075 break; 1076 } 1077 case MVT::f32: 1078 case MVT::f64: { 1079 std::vector<MVT::ValueType> Tys; 1080 Tys.push_back(MVT::f64); 1081 Tys.push_back(MVT::Other); 1082 Tys.push_back(MVT::Flag); 1083 std::vector<SDOperand> Ops; 1084 Ops.push_back(Chain); 1085 Ops.push_back(InFlag); 1086 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops); 1087 Chain = RetVal.getValue(1); 1088 InFlag = RetVal.getValue(2); 1089 if (X86ScalarSSE) { 1090 // FIXME:Currently the FST is flagged to the FP_GET_RESULT. This 1091 // shouldn't be necessary except for RFP cannot be live across 1092 // multiple blocks. When stackifier is fixed, they can be uncoupled. 1093 unsigned Size = MVT::getSizeInBits(MVT::f64)/8; 1094 MachineFunction &MF = DAG.getMachineFunction(); 1095 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 1096 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1097 Tys.clear(); 1098 Tys.push_back(MVT::Other); 1099 Ops.clear(); 1100 Ops.push_back(Chain); 1101 Ops.push_back(RetVal); 1102 Ops.push_back(StackSlot); 1103 Ops.push_back(DAG.getValueType(RetTyVT)); 1104 Ops.push_back(InFlag); 1105 Chain = DAG.getNode(X86ISD::FST, Tys, Ops); 1106 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot, 1107 DAG.getSrcValue(NULL)); 1108 Chain = RetVal.getValue(1); 1109 } 1110 1111 if (RetTyVT == MVT::f32 && !X86ScalarSSE) 1112 // FIXME: we would really like to remember that this FP_ROUND 1113 // operation is okay to eliminate if we allow excess FP precision. 1114 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); 1115 break; 1116 } 1117 } 1118 } 1119 1120 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain, 1121 DAG.getConstant(ArgOffset, getPointerTy()), 1122 DAG.getConstant(ArgOffset, getPointerTy())); 1123 return std::make_pair(RetVal, Chain); 1124 } else { 1125 std::vector<SDOperand> Ops; 1126 Ops.push_back(Chain); 1127 Ops.push_back(Callee); 1128 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1129 // Callee pops all arg values on the stack. 1130 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1131 1132 // Pass register arguments as needed. 1133 Ops.insert(Ops.end(), RegValuesToPass.begin(), RegValuesToPass.end()); 1134 1135 SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, 1136 RetVals, Ops); 1137 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall); 1138 1139 SDOperand ResultVal; 1140 switch (RetTyVT) { 1141 case MVT::isVoid: break; 1142 default: 1143 ResultVal = TheCall.getValue(1); 1144 break; 1145 case MVT::i1: 1146 case MVT::i8: 1147 case MVT::i16: 1148 ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1)); 1149 break; 1150 case MVT::f32: 1151 // FIXME: we would really like to remember that this FP_ROUND operation is 1152 // okay to eliminate if we allow excess FP precision. 1153 ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1)); 1154 break; 1155 case MVT::i64: 1156 ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1), 1157 TheCall.getValue(2)); 1158 break; 1159 } 1160 1161 return std::make_pair(ResultVal, Chain); 1162 } 1163} 1164 1165SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1166 if (ReturnAddrIndex == 0) { 1167 // Set up a frame object for the return address. 1168 MachineFunction &MF = DAG.getMachineFunction(); 1169 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1170 } 1171 1172 return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32); 1173} 1174 1175 1176 1177std::pair<SDOperand, SDOperand> X86TargetLowering:: 1178LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth, 1179 SelectionDAG &DAG) { 1180 SDOperand Result; 1181 if (Depth) // Depths > 0 not supported yet! 1182 Result = DAG.getConstant(0, getPointerTy()); 1183 else { 1184 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 1185 if (!isFrameAddress) 1186 // Just load the return address 1187 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI, 1188 DAG.getSrcValue(NULL)); 1189 else 1190 Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI, 1191 DAG.getConstant(4, MVT::i32)); 1192 } 1193 return std::make_pair(Result, Chain); 1194} 1195 1196/// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode 1197/// which corresponds to the condition code. 1198static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) { 1199 switch (X86CC) { 1200 default: assert(0 && "Unknown X86 conditional code!"); 1201 case X86ISD::COND_A: return X86::JA; 1202 case X86ISD::COND_AE: return X86::JAE; 1203 case X86ISD::COND_B: return X86::JB; 1204 case X86ISD::COND_BE: return X86::JBE; 1205 case X86ISD::COND_E: return X86::JE; 1206 case X86ISD::COND_G: return X86::JG; 1207 case X86ISD::COND_GE: return X86::JGE; 1208 case X86ISD::COND_L: return X86::JL; 1209 case X86ISD::COND_LE: return X86::JLE; 1210 case X86ISD::COND_NE: return X86::JNE; 1211 case X86ISD::COND_NO: return X86::JNO; 1212 case X86ISD::COND_NP: return X86::JNP; 1213 case X86ISD::COND_NS: return X86::JNS; 1214 case X86ISD::COND_O: return X86::JO; 1215 case X86ISD::COND_P: return X86::JP; 1216 case X86ISD::COND_S: return X86::JS; 1217 } 1218} 1219 1220/// getX86CC - do a one to one translation of a ISD::CondCode to the X86 1221/// specific condition code. It returns a X86ISD::COND_INVALID if it cannot 1222/// do a direct translation. 1223static unsigned getX86CC(SDOperand CC, bool isFP) { 1224 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 1225 unsigned X86CC = X86ISD::COND_INVALID; 1226 if (!isFP) { 1227 switch (SetCCOpcode) { 1228 default: break; 1229 case ISD::SETEQ: X86CC = X86ISD::COND_E; break; 1230 case ISD::SETGT: X86CC = X86ISD::COND_G; break; 1231 case ISD::SETGE: X86CC = X86ISD::COND_GE; break; 1232 case ISD::SETLT: X86CC = X86ISD::COND_L; break; 1233 case ISD::SETLE: X86CC = X86ISD::COND_LE; break; 1234 case ISD::SETNE: X86CC = X86ISD::COND_NE; break; 1235 case ISD::SETULT: X86CC = X86ISD::COND_B; break; 1236 case ISD::SETUGT: X86CC = X86ISD::COND_A; break; 1237 case ISD::SETULE: X86CC = X86ISD::COND_BE; break; 1238 case ISD::SETUGE: X86CC = X86ISD::COND_AE; break; 1239 } 1240 } else { 1241 // On a floating point condition, the flags are set as follows: 1242 // ZF PF CF op 1243 // 0 | 0 | 0 | X > Y 1244 // 0 | 0 | 1 | X < Y 1245 // 1 | 0 | 0 | X == Y 1246 // 1 | 1 | 1 | unordered 1247 switch (SetCCOpcode) { 1248 default: break; 1249 case ISD::SETUEQ: 1250 case ISD::SETEQ: X86CC = X86ISD::COND_E; break; 1251 case ISD::SETOGT: 1252 case ISD::SETGT: X86CC = X86ISD::COND_A; break; 1253 case ISD::SETOGE: 1254 case ISD::SETGE: X86CC = X86ISD::COND_AE; break; 1255 case ISD::SETULT: 1256 case ISD::SETLT: X86CC = X86ISD::COND_B; break; 1257 case ISD::SETULE: 1258 case ISD::SETLE: X86CC = X86ISD::COND_BE; break; 1259 case ISD::SETONE: 1260 case ISD::SETNE: X86CC = X86ISD::COND_NE; break; 1261 case ISD::SETUO: X86CC = X86ISD::COND_P; break; 1262 case ISD::SETO: X86CC = X86ISD::COND_NP; break; 1263 } 1264 } 1265 return X86CC; 1266} 1267 1268/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1269/// code. Current x86 isa includes the following FP cmov instructions: 1270/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1271static bool hasFPCMov(unsigned X86CC) { 1272 switch (X86CC) { 1273 default: 1274 return false; 1275 case X86ISD::COND_B: 1276 case X86ISD::COND_BE: 1277 case X86ISD::COND_E: 1278 case X86ISD::COND_P: 1279 case X86ISD::COND_A: 1280 case X86ISD::COND_AE: 1281 case X86ISD::COND_NE: 1282 case X86ISD::COND_NP: 1283 return true; 1284 } 1285} 1286 1287MachineBasicBlock * 1288X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI, 1289 MachineBasicBlock *BB) { 1290 switch (MI->getOpcode()) { 1291 default: assert(false && "Unexpected instr type to insert"); 1292 case X86::CMOV_FR32: 1293 case X86::CMOV_FR64: { 1294 // To "insert" a SELECT_CC instruction, we actually have to insert the diamond 1295 // control-flow pattern. The incoming instruction knows the destination vreg 1296 // to set, the condition code register to branch on, the true/false values to 1297 // select between, and a branch opcode to use. 1298 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1299 ilist<MachineBasicBlock>::iterator It = BB; 1300 ++It; 1301 1302 // thisMBB: 1303 // ... 1304 // TrueVal = ... 1305 // cmpTY ccX, r1, r2 1306 // bCC copy1MBB 1307 // fallthrough --> copy0MBB 1308 MachineBasicBlock *thisMBB = BB; 1309 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB); 1310 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB); 1311 unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue()); 1312 BuildMI(BB, Opc, 1).addMBB(sinkMBB); 1313 MachineFunction *F = BB->getParent(); 1314 F->getBasicBlockList().insert(It, copy0MBB); 1315 F->getBasicBlockList().insert(It, sinkMBB); 1316 // Update machine-CFG edges 1317 BB->addSuccessor(copy0MBB); 1318 BB->addSuccessor(sinkMBB); 1319 1320 // copy0MBB: 1321 // %FalseValue = ... 1322 // # fallthrough to sinkMBB 1323 BB = copy0MBB; 1324 1325 // Update machine-CFG edges 1326 BB->addSuccessor(sinkMBB); 1327 1328 // sinkMBB: 1329 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 1330 // ... 1331 BB = sinkMBB; 1332 BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg()) 1333 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 1334 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 1335 1336 delete MI; // The pseudo instruction is gone now. 1337 return BB; 1338 } 1339 1340 case X86::FP_TO_INT16_IN_MEM: 1341 case X86::FP_TO_INT32_IN_MEM: 1342 case X86::FP_TO_INT64_IN_MEM: { 1343 // Change the floating point control register to use "round towards zero" 1344 // mode when truncating to an integer value. 1345 MachineFunction *F = BB->getParent(); 1346 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 1347 addFrameReference(BuildMI(BB, X86::FNSTCW16m, 4), CWFrameIdx); 1348 1349 // Load the old value of the high byte of the control word... 1350 unsigned OldCW = 1351 F->getSSARegMap()->createVirtualRegister(X86::R16RegisterClass); 1352 addFrameReference(BuildMI(BB, X86::MOV16rm, 4, OldCW), CWFrameIdx); 1353 1354 // Set the high part to be round to zero... 1355 addFrameReference(BuildMI(BB, X86::MOV16mi, 5), CWFrameIdx).addImm(0xC7F); 1356 1357 // Reload the modified control word now... 1358 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx); 1359 1360 // Restore the memory image of control word to original value 1361 addFrameReference(BuildMI(BB, X86::MOV16mr, 5), CWFrameIdx).addReg(OldCW); 1362 1363 // Get the X86 opcode to use. 1364 unsigned Opc; 1365 switch (MI->getOpcode()) { 1366 case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break; 1367 case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break; 1368 case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break; 1369 } 1370 1371 X86AddressMode AM; 1372 MachineOperand &Op = MI->getOperand(0); 1373 if (Op.isRegister()) { 1374 AM.BaseType = X86AddressMode::RegBase; 1375 AM.Base.Reg = Op.getReg(); 1376 } else { 1377 AM.BaseType = X86AddressMode::FrameIndexBase; 1378 AM.Base.FrameIndex = Op.getFrameIndex(); 1379 } 1380 Op = MI->getOperand(1); 1381 if (Op.isImmediate()) 1382 AM.Scale = Op.getImmedValue(); 1383 Op = MI->getOperand(2); 1384 if (Op.isImmediate()) 1385 AM.IndexReg = Op.getImmedValue(); 1386 Op = MI->getOperand(3); 1387 if (Op.isGlobalAddress()) { 1388 AM.GV = Op.getGlobal(); 1389 } else { 1390 AM.Disp = Op.getImmedValue(); 1391 } 1392 addFullAddress(BuildMI(BB, Opc, 5), AM).addReg(MI->getOperand(4).getReg()); 1393 1394 // Reload the original control word now. 1395 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx); 1396 1397 delete MI; // The pseudo instruction is gone now. 1398 return BB; 1399 } 1400 } 1401} 1402 1403 1404//===----------------------------------------------------------------------===// 1405// X86 Custom Lowering Hooks 1406//===----------------------------------------------------------------------===// 1407 1408/// LowerOperation - Provide custom lowering hooks for some operations. 1409/// 1410SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 1411 switch (Op.getOpcode()) { 1412 default: assert(0 && "Should not custom lower this!"); 1413 case ISD::ADD_PARTS: 1414 case ISD::SUB_PARTS: { 1415 assert(Op.getNumOperands() == 4 && Op.getValueType() == MVT::i32 && 1416 "Not an i64 add/sub!"); 1417 bool isAdd = Op.getOpcode() == ISD::ADD_PARTS; 1418 std::vector<MVT::ValueType> Tys; 1419 Tys.push_back(MVT::i32); 1420 Tys.push_back(MVT::Flag); 1421 std::vector<SDOperand> Ops; 1422 Ops.push_back(Op.getOperand(0)); 1423 Ops.push_back(Op.getOperand(2)); 1424 SDOperand Lo = DAG.getNode(isAdd ? X86ISD::ADD_FLAG : X86ISD::SUB_FLAG, 1425 Tys, Ops); 1426 SDOperand Hi = DAG.getNode(isAdd ? X86ISD::ADC : X86ISD::SBB, MVT::i32, 1427 Op.getOperand(1), Op.getOperand(3), 1428 Lo.getValue(1)); 1429 Tys.clear(); 1430 Tys.push_back(MVT::i32); 1431 Tys.push_back(MVT::i32); 1432 Ops.clear(); 1433 Ops.push_back(Lo); 1434 Ops.push_back(Hi); 1435 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1436 } 1437 case ISD::SHL_PARTS: 1438 case ISD::SRA_PARTS: 1439 case ISD::SRL_PARTS: { 1440 assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 && 1441 "Not an i64 shift!"); 1442 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 1443 SDOperand ShOpLo = Op.getOperand(0); 1444 SDOperand ShOpHi = Op.getOperand(1); 1445 SDOperand ShAmt = Op.getOperand(2); 1446 SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi, 1447 DAG.getConstant(31, MVT::i32)) 1448 : DAG.getConstant(0, MVT::i32); 1449 1450 SDOperand Tmp2, Tmp3; 1451 if (Op.getOpcode() == ISD::SHL_PARTS) { 1452 Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt); 1453 Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt); 1454 } else { 1455 Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt); 1456 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SHL, MVT::i32, ShOpHi, ShAmt); 1457 } 1458 1459 SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag, 1460 ShAmt, DAG.getConstant(32, MVT::i8)); 1461 1462 SDOperand Hi, Lo; 1463 SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1464 1465 std::vector<MVT::ValueType> Tys; 1466 Tys.push_back(MVT::i32); 1467 Tys.push_back(MVT::Flag); 1468 std::vector<SDOperand> Ops; 1469 if (Op.getOpcode() == ISD::SHL_PARTS) { 1470 Ops.push_back(Tmp2); 1471 Ops.push_back(Tmp3); 1472 Ops.push_back(CC); 1473 Ops.push_back(InFlag); 1474 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1475 InFlag = Hi.getValue(1); 1476 1477 Ops.clear(); 1478 Ops.push_back(Tmp3); 1479 Ops.push_back(Tmp1); 1480 Ops.push_back(CC); 1481 Ops.push_back(InFlag); 1482 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1483 } else { 1484 Ops.push_back(Tmp2); 1485 Ops.push_back(Tmp3); 1486 Ops.push_back(CC); 1487 Ops.push_back(InFlag); 1488 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1489 InFlag = Lo.getValue(1); 1490 1491 Ops.clear(); 1492 Ops.push_back(Tmp3); 1493 Ops.push_back(Tmp1); 1494 Ops.push_back(CC); 1495 Ops.push_back(InFlag); 1496 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1497 } 1498 1499 Tys.clear(); 1500 Tys.push_back(MVT::i32); 1501 Tys.push_back(MVT::i32); 1502 Ops.clear(); 1503 Ops.push_back(Lo); 1504 Ops.push_back(Hi); 1505 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1506 } 1507 case ISD::SINT_TO_FP: { 1508 assert(Op.getValueType() == MVT::f64 && 1509 Op.getOperand(0).getValueType() <= MVT::i64 && 1510 Op.getOperand(0).getValueType() >= MVT::i16 && 1511 "Unknown SINT_TO_FP to lower!"); 1512 1513 SDOperand Result; 1514 MVT::ValueType SrcVT = Op.getOperand(0).getValueType(); 1515 unsigned Size = MVT::getSizeInBits(SrcVT)/8; 1516 MachineFunction &MF = DAG.getMachineFunction(); 1517 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 1518 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1519 SDOperand Chain = DAG.getNode(ISD::STORE, MVT::Other, 1520 DAG.getEntryNode(), Op.getOperand(0), 1521 StackSlot, DAG.getSrcValue(NULL)); 1522 1523 // Build the FILD 1524 std::vector<MVT::ValueType> Tys; 1525 Tys.push_back(MVT::f64); 1526 Tys.push_back(MVT::Flag); 1527 std::vector<SDOperand> Ops; 1528 Ops.push_back(Chain); 1529 Ops.push_back(StackSlot); 1530 Ops.push_back(DAG.getValueType(SrcVT)); 1531 Result = DAG.getNode(X86ISD::FILD, Tys, Ops); 1532 return Result; 1533 } 1534 case ISD::FP_TO_SINT: { 1535 assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 && 1536 Op.getOperand(0).getValueType() == MVT::f64 && 1537 "Unknown FP_TO_SINT to lower!"); 1538 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 1539 // stack slot. 1540 MachineFunction &MF = DAG.getMachineFunction(); 1541 unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8; 1542 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 1543 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1544 1545 unsigned Opc; 1546 switch (Op.getValueType()) { 1547 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 1548 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 1549 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 1550 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 1551 } 1552 1553 // Build the FP_TO_INT*_IN_MEM 1554 std::vector<SDOperand> Ops; 1555 Ops.push_back(DAG.getEntryNode()); 1556 Ops.push_back(Op.getOperand(0)); 1557 Ops.push_back(StackSlot); 1558 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops); 1559 1560 // Load the result. 1561 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, 1562 DAG.getSrcValue(NULL)); 1563 } 1564 case ISD::READCYCLECOUNTER: { 1565 std::vector<MVT::ValueType> Tys; 1566 Tys.push_back(MVT::Other); 1567 Tys.push_back(MVT::Flag); 1568 std::vector<SDOperand> Ops; 1569 Ops.push_back(Op.getOperand(0)); 1570 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops); 1571 Ops.clear(); 1572 Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1))); 1573 Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX, 1574 MVT::i32, Ops[0].getValue(2))); 1575 Ops.push_back(Ops[1].getValue(1)); 1576 Tys[0] = Tys[1] = MVT::i32; 1577 Tys.push_back(MVT::Other); 1578 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1579 } 1580 case ISD::SETCC: { 1581 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 1582 SDOperand CC = Op.getOperand(2); 1583 SDOperand Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1584 Op.getOperand(0), Op.getOperand(1)); 1585 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 1586 bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType()); 1587 unsigned X86CC = getX86CC(CC, isFP); 1588 if (X86CC != X86ISD::COND_INVALID) { 1589 return DAG.getNode(X86ISD::SETCC, MVT::i8, 1590 DAG.getConstant(X86CC, MVT::i8), Cond); 1591 } else { 1592 assert(isFP && "Illegal integer SetCC!"); 1593 1594 std::vector<MVT::ValueType> Tys; 1595 std::vector<SDOperand> Ops; 1596 switch (SetCCOpcode) { 1597 default: assert(false && "Illegal floating point SetCC!"); 1598 case ISD::SETOEQ: { // !PF & ZF 1599 Tys.push_back(MVT::i8); 1600 Tys.push_back(MVT::Flag); 1601 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1602 Ops.push_back(Cond); 1603 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1604 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1605 DAG.getConstant(X86ISD::COND_E, MVT::i8), 1606 Tmp1.getValue(1)); 1607 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1608 } 1609 case ISD::SETOLT: { // !PF & CF 1610 Tys.push_back(MVT::i8); 1611 Tys.push_back(MVT::Flag); 1612 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1613 Ops.push_back(Cond); 1614 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1615 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1616 DAG.getConstant(X86ISD::COND_B, MVT::i8), 1617 Tmp1.getValue(1)); 1618 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1619 } 1620 case ISD::SETOLE: { // !PF & (CF || ZF) 1621 Tys.push_back(MVT::i8); 1622 Tys.push_back(MVT::Flag); 1623 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1624 Ops.push_back(Cond); 1625 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1626 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1627 DAG.getConstant(X86ISD::COND_BE, MVT::i8), 1628 Tmp1.getValue(1)); 1629 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1630 } 1631 case ISD::SETUGT: { // PF | (!ZF & !CF) 1632 Tys.push_back(MVT::i8); 1633 Tys.push_back(MVT::Flag); 1634 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1635 Ops.push_back(Cond); 1636 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1637 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1638 DAG.getConstant(X86ISD::COND_A, MVT::i8), 1639 Tmp1.getValue(1)); 1640 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1641 } 1642 case ISD::SETUGE: { // PF | !CF 1643 Tys.push_back(MVT::i8); 1644 Tys.push_back(MVT::Flag); 1645 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1646 Ops.push_back(Cond); 1647 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1648 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1649 DAG.getConstant(X86ISD::COND_AE, MVT::i8), 1650 Tmp1.getValue(1)); 1651 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1652 } 1653 case ISD::SETUNE: { // PF | !ZF 1654 Tys.push_back(MVT::i8); 1655 Tys.push_back(MVT::Flag); 1656 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1657 Ops.push_back(Cond); 1658 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1659 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1660 DAG.getConstant(X86ISD::COND_NE, MVT::i8), 1661 Tmp1.getValue(1)); 1662 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1663 } 1664 } 1665 } 1666 } 1667 case ISD::SELECT: { 1668 MVT::ValueType VT = Op.getValueType(); 1669 bool isFP = MVT::isFloatingPoint(VT); 1670 bool isFPStack = isFP && (X86Vector < SSE2); 1671 bool isFPSSE = isFP && (X86Vector >= SSE2); 1672 bool addTest = false; 1673 SDOperand Op0 = Op.getOperand(0); 1674 SDOperand Cond, CC; 1675 if (Op0.getOpcode() == X86ISD::SETCC) { 1676 // If condition flag is set by a X86ISD::CMP, then make a copy of it 1677 // (since flag operand cannot be shared). If the X86ISD::SETCC does not 1678 // have another use it will be eliminated. 1679 // If the X86ISD::SETCC has more than one use, then it's probably better 1680 // to use a test instead of duplicating the X86ISD::CMP (for register 1681 // pressure reason). 1682 if (Op0.hasOneUse() && Op0.getOperand(1).getOpcode() == X86ISD::CMP) { 1683 CC = Op0.getOperand(0); 1684 Cond = Op0.getOperand(1); 1685 addTest = 1686 isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 1687 } else 1688 addTest = true; 1689 } else if (Op0.getOpcode() == ISD::SETCC) { 1690 CC = Op0.getOperand(2); 1691 bool isFP = MVT::isFloatingPoint(Op0.getOperand(1).getValueType()); 1692 unsigned X86CC = getX86CC(CC, isFP); 1693 CC = DAG.getConstant(X86CC, MVT::i8); 1694 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1695 Op0.getOperand(0), Op0.getOperand(1)); 1696 addTest = true; 1697 } else 1698 addTest = true; 1699 1700 if (addTest) { 1701 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1702 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0); 1703 } 1704 1705 std::vector<MVT::ValueType> Tys; 1706 Tys.push_back(Op.getValueType()); 1707 Tys.push_back(MVT::Flag); 1708 std::vector<SDOperand> Ops; 1709 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 1710 // condition is true. 1711 Ops.push_back(Op.getOperand(2)); 1712 Ops.push_back(Op.getOperand(1)); 1713 Ops.push_back(CC); 1714 Ops.push_back(Cond); 1715 return DAG.getNode(X86ISD::CMOV, Tys, Ops); 1716 } 1717 case ISD::BRCOND: { 1718 bool addTest = false; 1719 SDOperand Cond = Op.getOperand(1); 1720 SDOperand Dest = Op.getOperand(2); 1721 SDOperand CC; 1722 if (Cond.getOpcode() == X86ISD::SETCC) { 1723 // If condition flag is set by a X86ISD::CMP, then make a copy of it 1724 // (since flag operand cannot be shared). If the X86ISD::SETCC does not 1725 // have another use it will be eliminated. 1726 // If the X86ISD::SETCC has more than one use, then it's probably better 1727 // to use a test instead of duplicating the X86ISD::CMP (for register 1728 // pressure reason). 1729 if (Cond.hasOneUse() && Cond.getOperand(1).getOpcode() == X86ISD::CMP) { 1730 CC = Cond.getOperand(0); 1731 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1732 Cond.getOperand(1).getOperand(0), 1733 Cond.getOperand(1).getOperand(1)); 1734 } else 1735 addTest = true; 1736 } else if (Cond.getOpcode() == ISD::SETCC) { 1737 CC = Cond.getOperand(2); 1738 bool isFP = MVT::isFloatingPoint(Cond.getOperand(1).getValueType()); 1739 unsigned X86CC = getX86CC(CC, isFP); 1740 CC = DAG.getConstant(X86CC, MVT::i8); 1741 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1742 Cond.getOperand(0), Cond.getOperand(1)); 1743 } else 1744 addTest = true; 1745 1746 if (addTest) { 1747 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1748 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond); 1749 } 1750 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 1751 Op.getOperand(0), Op.getOperand(2), CC, Cond); 1752 } 1753 case ISD::RET: { 1754 // Can only be return void. 1755 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0), 1756 DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 1757 } 1758 case ISD::MEMSET: { 1759 SDOperand InFlag; 1760 SDOperand Chain = Op.getOperand(0); 1761 unsigned Align = 1762 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); 1763 if (Align == 0) Align = 1; 1764 1765 MVT::ValueType AVT; 1766 SDOperand Count; 1767 if (ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2))) { 1768 unsigned ValReg; 1769 unsigned Val = ValC->getValue() & 255; 1770 1771 // If the value is a constant, then we can potentially use larger sets. 1772 switch (Align & 3) { 1773 case 2: // WORD aligned 1774 AVT = MVT::i16; 1775 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1776 Count = DAG.getConstant(I->getValue() / 2, MVT::i32); 1777 else 1778 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1779 DAG.getConstant(1, MVT::i8)); 1780 Val = (Val << 8) | Val; 1781 ValReg = X86::AX; 1782 break; 1783 case 0: // DWORD aligned 1784 AVT = MVT::i32; 1785 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1786 Count = DAG.getConstant(I->getValue() / 4, MVT::i32); 1787 else 1788 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1789 DAG.getConstant(2, MVT::i8)); 1790 Val = (Val << 8) | Val; 1791 Val = (Val << 16) | Val; 1792 ValReg = X86::EAX; 1793 break; 1794 default: // Byte aligned 1795 AVT = MVT::i8; 1796 Count = Op.getOperand(3); 1797 ValReg = X86::AL; 1798 break; 1799 } 1800 1801 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 1802 InFlag); 1803 InFlag = Chain.getValue(1); 1804 } else { 1805 AVT = MVT::i8; 1806 Count = Op.getOperand(3); 1807 Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag); 1808 InFlag = Chain.getValue(1); 1809 } 1810 1811 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag); 1812 InFlag = Chain.getValue(1); 1813 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag); 1814 InFlag = Chain.getValue(1); 1815 1816 return DAG.getNode(X86ISD::REP_STOS, MVT::Other, Chain, 1817 DAG.getValueType(AVT), InFlag); 1818 } 1819 case ISD::MEMCPY: { 1820 SDOperand Chain = Op.getOperand(0); 1821 unsigned Align = 1822 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); 1823 if (Align == 0) Align = 1; 1824 1825 MVT::ValueType AVT; 1826 SDOperand Count; 1827 switch (Align & 3) { 1828 case 2: // WORD aligned 1829 AVT = MVT::i16; 1830 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1831 Count = DAG.getConstant(I->getValue() / 2, MVT::i32); 1832 else 1833 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1834 DAG.getConstant(1, MVT::i8)); 1835 break; 1836 case 0: // DWORD aligned 1837 AVT = MVT::i32; 1838 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1839 Count = DAG.getConstant(I->getValue() / 4, MVT::i32); 1840 else 1841 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1842 DAG.getConstant(2, MVT::i8)); 1843 break; 1844 default: // Byte aligned 1845 AVT = MVT::i8; 1846 Count = Op.getOperand(3); 1847 break; 1848 } 1849 1850 SDOperand InFlag; 1851 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag); 1852 InFlag = Chain.getValue(1); 1853 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag); 1854 InFlag = Chain.getValue(1); 1855 Chain = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag); 1856 InFlag = Chain.getValue(1); 1857 1858 return DAG.getNode(X86ISD::REP_MOVS, MVT::Other, Chain, 1859 DAG.getValueType(AVT), InFlag); 1860 } 1861 case ISD::GlobalAddress: { 1862 SDOperand Result; 1863 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 1864 // For Darwin, external and weak symbols are indirect, so we want to load 1865 // the value at address GV, not the value of GV itself. This means that 1866 // the GlobalAddress must be in the base or index register of the address, 1867 // not the GV offset field. 1868 if (getTargetMachine(). 1869 getSubtarget<X86Subtarget>().getIndirectExternAndWeakGlobals() && 1870 (GV->hasWeakLinkage() || GV->isExternal())) 1871 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), 1872 DAG.getTargetGlobalAddress(GV, getPointerTy()), 1873 DAG.getSrcValue(NULL)); 1874 return Result; 1875 } 1876 } 1877} 1878 1879const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 1880 switch (Opcode) { 1881 default: return NULL; 1882 case X86ISD::ADD_FLAG: return "X86ISD::ADD_FLAG"; 1883 case X86ISD::SUB_FLAG: return "X86ISD::SUB_FLAG"; 1884 case X86ISD::ADC: return "X86ISD::ADC"; 1885 case X86ISD::SBB: return "X86ISD::SBB"; 1886 case X86ISD::SHLD: return "X86ISD::SHLD"; 1887 case X86ISD::SHRD: return "X86ISD::SHRD"; 1888 case X86ISD::FILD: return "X86ISD::FILD"; 1889 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 1890 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 1891 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 1892 case X86ISD::FLD: return "X86ISD::FLD"; 1893 case X86ISD::FST: return "X86ISD::FST"; 1894 case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT"; 1895 case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT"; 1896 case X86ISD::CALL: return "X86ISD::CALL"; 1897 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 1898 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 1899 case X86ISD::CMP: return "X86ISD::CMP"; 1900 case X86ISD::TEST: return "X86ISD::TEST"; 1901 case X86ISD::SETCC: return "X86ISD::SETCC"; 1902 case X86ISD::CMOV: return "X86ISD::CMOV"; 1903 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 1904 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 1905 case X86ISD::REP_STOS: return "X86ISD::RET_STOS"; 1906 case X86ISD::REP_MOVS: return "X86ISD::RET_MOVS"; 1907 } 1908} 1909 1910bool X86TargetLowering::isMaskedValueZeroForTargetNode(const SDOperand &Op, 1911 uint64_t Mask) const { 1912 1913 unsigned Opc = Op.getOpcode(); 1914 1915 switch (Opc) { 1916 default: 1917 assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node"); 1918 break; 1919 case X86ISD::SETCC: return (Mask & 1) == 0; 1920 } 1921 1922 return false; 1923} 1924