X86ISelLowering.cpp revision 0d718e9afca29b047ae1c77f8c6afc1b70759e0b
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by Chris Lattner and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the interfaces that X86 uses to lower LLVM code into a 11// selection DAG. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86ISelLowering.h" 18#include "X86TargetMachine.h" 19#include "llvm/CallingConv.h" 20#include "llvm/Function.h" 21#include "llvm/CodeGen/MachineFrameInfo.h" 22#include "llvm/CodeGen/MachineFunction.h" 23#include "llvm/CodeGen/MachineInstrBuilder.h" 24#include "llvm/CodeGen/SelectionDAG.h" 25#include "llvm/CodeGen/SSARegMap.h" 26#include "llvm/Target/TargetOptions.h" 27using namespace llvm; 28 29// FIXME: temporary. 30#include "llvm/Support/CommandLine.h" 31static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden, 32 cl::desc("Enable fastcc on X86")); 33 34X86TargetLowering::X86TargetLowering(TargetMachine &TM) 35 : TargetLowering(TM) { 36 // Set up the TargetLowering object. 37 38 // X86 is weird, it always uses i8 for shift amounts and setcc results. 39 setShiftAmountType(MVT::i8); 40 setSetCCResultType(MVT::i8); 41 setSetCCResultContents(ZeroOrOneSetCCResult); 42 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 43 setStackPointerRegisterToSaveRestore(X86::ESP); 44 45 // Set up the register classes. 46 addRegisterClass(MVT::i8, X86::R8RegisterClass); 47 addRegisterClass(MVT::i16, X86::R16RegisterClass); 48 addRegisterClass(MVT::i32, X86::R32RegisterClass); 49 50 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 51 // operation. 52 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 53 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 54 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 55 56 if (X86ScalarSSE) 57 // No SSE i64 SINT_TO_FP, so expand i32 UINT_TO_FP instead. 58 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); 59 else 60 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 61 62 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 63 // this operation. 64 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 65 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 66 67 if (!X86ScalarSSE) { 68 // We can handle SINT_TO_FP and FP_TO_SINT from/TO i64 even though i64 69 // isn't legal. 70 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 71 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 72 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 73 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 74 } 75 76 // Handle FP_TO_UINT by promoting the destination to a larger signed 77 // conversion. 78 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 79 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 80 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 81 82 if (!X86ScalarSSE) 83 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 84 85 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 86 // this operation. 87 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 88 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 89 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 90 91 setOperationAction(ISD::BIT_CONVERT, MVT::f32, Expand); 92 setOperationAction(ISD::BIT_CONVERT, MVT::i32, Expand); 93 94 if (X86DAGIsel) { 95 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 96 } 97 setOperationAction(ISD::BRCONDTWOWAY , MVT::Other, Expand); 98 setOperationAction(ISD::BRTWOWAY_CC , MVT::Other, Expand); 99 setOperationAction(ISD::MEMMOVE , MVT::Other, Expand); 100 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand); 101 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand); 102 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 103 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 104 setOperationAction(ISD::SEXTLOAD , MVT::i1 , Expand); 105 setOperationAction(ISD::FREM , MVT::f64 , Expand); 106 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 107 setOperationAction(ISD::CTTZ , MVT::i8 , Expand); 108 setOperationAction(ISD::CTLZ , MVT::i8 , Expand); 109 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 110 setOperationAction(ISD::CTTZ , MVT::i16 , Expand); 111 setOperationAction(ISD::CTLZ , MVT::i16 , Expand); 112 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 113 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 114 setOperationAction(ISD::CTLZ , MVT::i32 , Expand); 115 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 116 117 if (!X86DAGIsel) { 118 setOperationAction(ISD::BSWAP , MVT::i32 , Expand); 119 setOperationAction(ISD::ROTL , MVT::i8 , Expand); 120 setOperationAction(ISD::ROTR , MVT::i8 , Expand); 121 setOperationAction(ISD::ROTL , MVT::i16 , Expand); 122 setOperationAction(ISD::ROTR , MVT::i16 , Expand); 123 setOperationAction(ISD::ROTL , MVT::i32 , Expand); 124 setOperationAction(ISD::ROTR , MVT::i32 , Expand); 125 } 126 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 127 128 setOperationAction(ISD::READIO , MVT::i1 , Expand); 129 setOperationAction(ISD::READIO , MVT::i8 , Expand); 130 setOperationAction(ISD::READIO , MVT::i16 , Expand); 131 setOperationAction(ISD::READIO , MVT::i32 , Expand); 132 setOperationAction(ISD::WRITEIO , MVT::i1 , Expand); 133 setOperationAction(ISD::WRITEIO , MVT::i8 , Expand); 134 setOperationAction(ISD::WRITEIO , MVT::i16 , Expand); 135 setOperationAction(ISD::WRITEIO , MVT::i32 , Expand); 136 137 // These should be promoted to a larger select which is supported. 138 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 139 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 140 if (X86DAGIsel) { 141 // X86 wants to expand cmov itself. 142 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 143 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 144 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 145 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 146 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 147 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 148 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 149 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 150 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 151 // X86 ret instruction may pop stack. 152 setOperationAction(ISD::RET , MVT::Other, Custom); 153 // Darwin ABI issue. 154 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 155 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 156 setOperationAction(ISD::ADD_PARTS , MVT::i32 , Custom); 157 setOperationAction(ISD::SUB_PARTS , MVT::i32 , Custom); 158 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 159 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 160 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 161 // X86 wants to expand memset / memcpy itself. 162 setOperationAction(ISD::MEMSET , MVT::Other, Custom); 163 setOperationAction(ISD::MEMCPY , MVT::Other, Custom); 164 } 165 166 // We don't have line number support yet. 167 setOperationAction(ISD::LOCATION, MVT::Other, Expand); 168 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand); 169 setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand); 170 171 // Expand to the default code. 172 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 173 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 174 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand); 175 176 if (X86ScalarSSE) { 177 // Set up the FP register classes. 178 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 179 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 180 181 // SSE has no load+extend ops 182 setOperationAction(ISD::EXTLOAD, MVT::f32, Expand); 183 setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand); 184 185 // SSE has no i16 to fp conversion, only i32 186 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); 187 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); 188 189 // Expand FP_TO_UINT into a select. 190 // FIXME: We would like to use a Custom expander here eventually to do 191 // the optimal thing for SSE vs. the default expansion in the legalizer. 192 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 193 194 // We don't support sin/cos/sqrt/fmod 195 setOperationAction(ISD::FSIN , MVT::f64, Expand); 196 setOperationAction(ISD::FCOS , MVT::f64, Expand); 197 setOperationAction(ISD::FABS , MVT::f64, Expand); 198 setOperationAction(ISD::FNEG , MVT::f64, Expand); 199 setOperationAction(ISD::FREM , MVT::f64, Expand); 200 setOperationAction(ISD::FSIN , MVT::f32, Expand); 201 setOperationAction(ISD::FCOS , MVT::f32, Expand); 202 setOperationAction(ISD::FABS , MVT::f32, Expand); 203 setOperationAction(ISD::FNEG , MVT::f32, Expand); 204 setOperationAction(ISD::FREM , MVT::f32, Expand); 205 206 addLegalFPImmediate(+0.0); // xorps / xorpd 207 } else { 208 // Set up the FP register classes. 209 addRegisterClass(MVT::f64, X86::RFPRegisterClass); 210 211 if (X86DAGIsel) { 212 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); 213 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 214 } 215 216 if (!UnsafeFPMath) { 217 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 218 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 219 } 220 221 addLegalFPImmediate(+0.0); // FLD0 222 addLegalFPImmediate(+1.0); // FLD1 223 addLegalFPImmediate(-0.0); // FLD0/FCHS 224 addLegalFPImmediate(-1.0); // FLD1/FCHS 225 } 226 computeRegisterProperties(); 227 228 maxStoresPerMemSet = 8; // For %llvm.memset -> sequence of stores 229 maxStoresPerMemCpy = 8; // For %llvm.memcpy -> sequence of stores 230 maxStoresPerMemMove = 8; // For %llvm.memmove -> sequence of stores 231 allowUnalignedMemoryAccesses = true; // x86 supports it! 232} 233 234std::vector<SDOperand> 235X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { 236 if (F.getCallingConv() == CallingConv::Fast && EnableFastCC) 237 return LowerFastCCArguments(F, DAG); 238 return LowerCCCArguments(F, DAG); 239} 240 241std::pair<SDOperand, SDOperand> 242X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, 243 bool isVarArg, unsigned CallingConv, 244 bool isTailCall, 245 SDOperand Callee, ArgListTy &Args, 246 SelectionDAG &DAG) { 247 assert((!isVarArg || CallingConv == CallingConv::C) && 248 "Only C takes varargs!"); 249 250 // If the callee is a GlobalAddress node (quite common, every direct call is) 251 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 252 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 253 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); 254 else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) 255 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); 256 257 if (CallingConv == CallingConv::Fast && EnableFastCC) 258 return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG); 259 return LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG); 260} 261 262SDOperand X86TargetLowering::LowerReturnTo(SDOperand Chain, SDOperand Op, 263 SelectionDAG &DAG) { 264 if (!X86DAGIsel) 265 return DAG.getNode(ISD::RET, MVT::Other, Chain, Op); 266 267 SDOperand Copy; 268 MVT::ValueType OpVT = Op.getValueType(); 269 switch (OpVT) { 270 default: assert(0 && "Unknown type to return!"); 271 case MVT::i32: 272 Copy = DAG.getCopyToReg(Chain, X86::EAX, Op, SDOperand()); 273 break; 274 case MVT::i64: { 275 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op, 276 DAG.getConstant(1, MVT::i32)); 277 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op, 278 DAG.getConstant(0, MVT::i32)); 279 Copy = DAG.getCopyToReg(Chain, X86::EDX, Hi, SDOperand()); 280 Copy = DAG.getCopyToReg(Copy, X86::EAX, Lo, Copy.getValue(1)); 281 break; 282 } 283 case MVT::f32: 284 case MVT::f64: 285 if (!X86ScalarSSE) { 286 std::vector<MVT::ValueType> Tys; 287 Tys.push_back(MVT::Other); 288 Tys.push_back(MVT::Flag); 289 std::vector<SDOperand> Ops; 290 Ops.push_back(Chain); 291 Ops.push_back(Op); 292 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops); 293 } else { 294 // Spill the value to memory and reload it into top of stack. 295 unsigned Size = MVT::getSizeInBits(OpVT)/8; 296 MachineFunction &MF = DAG.getMachineFunction(); 297 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 298 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 299 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Op, 300 StackSlot, DAG.getSrcValue(NULL)); 301 std::vector<MVT::ValueType> Tys; 302 Tys.push_back(MVT::f64); 303 Tys.push_back(MVT::Other); 304 std::vector<SDOperand> Ops; 305 Ops.push_back(Chain); 306 Ops.push_back(StackSlot); 307 Ops.push_back(DAG.getValueType(OpVT)); 308 Copy = DAG.getNode(X86ISD::FLD, Tys, Ops); 309 Tys.clear(); 310 Tys.push_back(MVT::Other); 311 Tys.push_back(MVT::Flag); 312 Ops.clear(); 313 Ops.push_back(Copy.getValue(1)); 314 Ops.push_back(Copy); 315 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops); 316 } 317 break; 318 } 319 320 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, 321 Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16), 322 Copy.getValue(1)); 323} 324 325//===----------------------------------------------------------------------===// 326// C Calling Convention implementation 327//===----------------------------------------------------------------------===// 328 329std::vector<SDOperand> 330X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) { 331 std::vector<SDOperand> ArgValues; 332 333 MachineFunction &MF = DAG.getMachineFunction(); 334 MachineFrameInfo *MFI = MF.getFrameInfo(); 335 336 // Add DAG nodes to load the arguments... On entry to a function on the X86, 337 // the stack frame looks like this: 338 // 339 // [ESP] -- return address 340 // [ESP + 4] -- first argument (leftmost lexically) 341 // [ESP + 8] -- second argument, if first argument is four bytes in size 342 // ... 343 // 344 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot 345 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) { 346 MVT::ValueType ObjectVT = getValueType(I->getType()); 347 unsigned ArgIncrement = 4; 348 unsigned ObjSize; 349 switch (ObjectVT) { 350 default: assert(0 && "Unhandled argument type!"); 351 case MVT::i1: 352 case MVT::i8: ObjSize = 1; break; 353 case MVT::i16: ObjSize = 2; break; 354 case MVT::i32: ObjSize = 4; break; 355 case MVT::i64: ObjSize = ArgIncrement = 8; break; 356 case MVT::f32: ObjSize = 4; break; 357 case MVT::f64: ObjSize = ArgIncrement = 8; break; 358 } 359 // Create the frame index object for this incoming parameter... 360 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); 361 362 // Create the SelectionDAG nodes corresponding to a load from this parameter 363 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 364 365 // Don't codegen dead arguments. FIXME: remove this check when we can nuke 366 // dead loads. 367 SDOperand ArgValue; 368 if (!I->use_empty()) 369 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN, 370 DAG.getSrcValue(NULL)); 371 else { 372 if (MVT::isInteger(ObjectVT)) 373 ArgValue = DAG.getConstant(0, ObjectVT); 374 else 375 ArgValue = DAG.getConstantFP(0, ObjectVT); 376 } 377 ArgValues.push_back(ArgValue); 378 379 ArgOffset += ArgIncrement; // Move on to the next argument... 380 } 381 382 // If the function takes variable number of arguments, make a frame index for 383 // the start of the first vararg value... for expansion of llvm.va_start. 384 if (F.isVarArg()) 385 VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset); 386 ReturnAddrIndex = 0; // No return address slot generated yet. 387 BytesToPopOnReturn = 0; // Callee pops nothing. 388 BytesCallerReserves = ArgOffset; 389 390 // Finally, inform the code generator which regs we return values in. 391 switch (getValueType(F.getReturnType())) { 392 default: assert(0 && "Unknown type!"); 393 case MVT::isVoid: break; 394 case MVT::i1: 395 case MVT::i8: 396 case MVT::i16: 397 case MVT::i32: 398 MF.addLiveOut(X86::EAX); 399 break; 400 case MVT::i64: 401 MF.addLiveOut(X86::EAX); 402 MF.addLiveOut(X86::EDX); 403 break; 404 case MVT::f32: 405 case MVT::f64: 406 MF.addLiveOut(X86::ST0); 407 break; 408 } 409 return ArgValues; 410} 411 412std::pair<SDOperand, SDOperand> 413X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy, 414 bool isVarArg, bool isTailCall, 415 SDOperand Callee, ArgListTy &Args, 416 SelectionDAG &DAG) { 417 // Count how many bytes are to be pushed on the stack. 418 unsigned NumBytes = 0; 419 420 if (Args.empty()) { 421 // Save zero bytes. 422 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 423 DAG.getConstant(0, getPointerTy())); 424 } else { 425 for (unsigned i = 0, e = Args.size(); i != e; ++i) 426 switch (getValueType(Args[i].second)) { 427 default: assert(0 && "Unknown value type!"); 428 case MVT::i1: 429 case MVT::i8: 430 case MVT::i16: 431 case MVT::i32: 432 case MVT::f32: 433 NumBytes += 4; 434 break; 435 case MVT::i64: 436 case MVT::f64: 437 NumBytes += 8; 438 break; 439 } 440 441 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 442 DAG.getConstant(NumBytes, getPointerTy())); 443 444 // Arguments go on the stack in reverse order, as specified by the ABI. 445 unsigned ArgOffset = 0; 446 SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32); 447 std::vector<SDOperand> Stores; 448 449 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 450 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 451 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 452 453 switch (getValueType(Args[i].second)) { 454 default: assert(0 && "Unexpected ValueType for argument!"); 455 case MVT::i1: 456 case MVT::i8: 457 case MVT::i16: 458 // Promote the integer to 32 bits. If the input type is signed use a 459 // sign extend, otherwise use a zero extend. 460 if (Args[i].second->isSigned()) 461 Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first); 462 else 463 Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first); 464 465 // FALL THROUGH 466 case MVT::i32: 467 case MVT::f32: 468 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 469 Args[i].first, PtrOff, 470 DAG.getSrcValue(NULL))); 471 ArgOffset += 4; 472 break; 473 case MVT::i64: 474 case MVT::f64: 475 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 476 Args[i].first, PtrOff, 477 DAG.getSrcValue(NULL))); 478 ArgOffset += 8; 479 break; 480 } 481 } 482 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); 483 } 484 485 std::vector<MVT::ValueType> RetVals; 486 MVT::ValueType RetTyVT = getValueType(RetTy); 487 RetVals.push_back(MVT::Other); 488 489 // The result values produced have to be legal. Promote the result. 490 switch (RetTyVT) { 491 case MVT::isVoid: break; 492 default: 493 RetVals.push_back(RetTyVT); 494 break; 495 case MVT::i1: 496 case MVT::i8: 497 case MVT::i16: 498 RetVals.push_back(MVT::i32); 499 break; 500 case MVT::f32: 501 if (X86ScalarSSE) 502 RetVals.push_back(MVT::f32); 503 else 504 RetVals.push_back(MVT::f64); 505 break; 506 case MVT::i64: 507 RetVals.push_back(MVT::i32); 508 RetVals.push_back(MVT::i32); 509 break; 510 } 511 512 if (X86DAGIsel) { 513 std::vector<MVT::ValueType> NodeTys; 514 NodeTys.push_back(MVT::Other); // Returns a chain 515 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 516 std::vector<SDOperand> Ops; 517 Ops.push_back(Chain); 518 Ops.push_back(Callee); 519 520 // FIXME: Do not generate X86ISD::TAILCALL for now. 521 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops); 522 SDOperand InFlag = Chain.getValue(1); 523 524 NodeTys.clear(); 525 NodeTys.push_back(MVT::Other); // Returns a chain 526 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 527 Ops.clear(); 528 Ops.push_back(Chain); 529 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); 530 Ops.push_back(DAG.getConstant(0, getPointerTy())); 531 Ops.push_back(InFlag); 532 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, Ops); 533 InFlag = Chain.getValue(1); 534 535 SDOperand RetVal; 536 if (RetTyVT != MVT::isVoid) { 537 switch (RetTyVT) { 538 default: assert(0 && "Unknown value type to return!"); 539 case MVT::i1: 540 case MVT::i8: 541 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag); 542 Chain = RetVal.getValue(1); 543 if (RetTyVT == MVT::i1) 544 RetVal = DAG.getNode(ISD::TRUNCATE, MVT::i1, RetVal); 545 break; 546 case MVT::i16: 547 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); 548 Chain = RetVal.getValue(1); 549 break; 550 case MVT::i32: 551 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 552 Chain = RetVal.getValue(1); 553 break; 554 case MVT::i64: { 555 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 556 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32, 557 Lo.getValue(2)); 558 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); 559 Chain = Hi.getValue(1); 560 break; 561 } 562 case MVT::f32: 563 case MVT::f64: { 564 std::vector<MVT::ValueType> Tys; 565 Tys.push_back(MVT::f64); 566 Tys.push_back(MVT::Other); 567 Tys.push_back(MVT::Flag); 568 std::vector<SDOperand> Ops; 569 Ops.push_back(Chain); 570 Ops.push_back(InFlag); 571 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops); 572 Chain = RetVal.getValue(1); 573 InFlag = RetVal.getValue(2); 574 if (X86ScalarSSE) { 575 // FIXME:Currently the FST is flagged to the FP_GET_RESULT. This 576 // shouldn't be necessary except for RFP cannot be live across 577 // multiple blocks. When stackifier is fixed, they can be uncoupled. 578 unsigned Size = MVT::getSizeInBits(MVT::f64)/8; 579 MachineFunction &MF = DAG.getMachineFunction(); 580 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 581 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 582 Tys.clear(); 583 Tys.push_back(MVT::Other); 584 Ops.clear(); 585 Ops.push_back(Chain); 586 Ops.push_back(RetVal); 587 Ops.push_back(StackSlot); 588 Ops.push_back(DAG.getValueType(RetTyVT)); 589 Ops.push_back(InFlag); 590 Chain = DAG.getNode(X86ISD::FST, Tys, Ops); 591 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot, 592 DAG.getSrcValue(NULL)); 593 Chain = RetVal.getValue(1); 594 } 595 596 if (RetTyVT == MVT::f32 && !X86ScalarSSE) 597 // FIXME: we would really like to remember that this FP_ROUND 598 // operation is okay to eliminate if we allow excess FP precision. 599 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); 600 break; 601 } 602 } 603 } 604 605 return std::make_pair(RetVal, Chain); 606 } else { 607 std::vector<SDOperand> Ops; 608 Ops.push_back(Chain); 609 Ops.push_back(Callee); 610 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); 611 Ops.push_back(DAG.getConstant(0, getPointerTy())); 612 613 SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, 614 RetVals, Ops); 615 616 SDOperand ResultVal; 617 switch (RetTyVT) { 618 case MVT::isVoid: break; 619 default: 620 ResultVal = TheCall.getValue(1); 621 break; 622 case MVT::i1: 623 case MVT::i8: 624 case MVT::i16: 625 ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1)); 626 break; 627 case MVT::f32: 628 // FIXME: we would really like to remember that this FP_ROUND operation is 629 // okay to eliminate if we allow excess FP precision. 630 ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1)); 631 break; 632 case MVT::i64: 633 ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1), 634 TheCall.getValue(2)); 635 break; 636 } 637 638 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall); 639 return std::make_pair(ResultVal, Chain); 640 } 641} 642 643SDOperand 644X86TargetLowering::LowerVAStart(SDOperand Chain, SDOperand VAListP, 645 Value *VAListV, SelectionDAG &DAG) { 646 // vastart just stores the address of the VarArgsFrameIndex slot. 647 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32); 648 return DAG.getNode(ISD::STORE, MVT::Other, Chain, FR, VAListP, 649 DAG.getSrcValue(VAListV)); 650} 651 652 653std::pair<SDOperand,SDOperand> 654X86TargetLowering::LowerVAArg(SDOperand Chain, SDOperand VAListP, 655 Value *VAListV, const Type *ArgTy, 656 SelectionDAG &DAG) { 657 MVT::ValueType ArgVT = getValueType(ArgTy); 658 SDOperand Val = DAG.getLoad(MVT::i32, Chain, 659 VAListP, DAG.getSrcValue(VAListV)); 660 SDOperand Result = DAG.getLoad(ArgVT, Chain, Val, 661 DAG.getSrcValue(NULL)); 662 unsigned Amt; 663 if (ArgVT == MVT::i32) 664 Amt = 4; 665 else { 666 assert((ArgVT == MVT::i64 || ArgVT == MVT::f64) && 667 "Other types should have been promoted for varargs!"); 668 Amt = 8; 669 } 670 Val = DAG.getNode(ISD::ADD, Val.getValueType(), Val, 671 DAG.getConstant(Amt, Val.getValueType())); 672 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, 673 Val, VAListP, DAG.getSrcValue(VAListV)); 674 return std::make_pair(Result, Chain); 675} 676 677//===----------------------------------------------------------------------===// 678// Fast Calling Convention implementation 679//===----------------------------------------------------------------------===// 680// 681// The X86 'fast' calling convention passes up to two integer arguments in 682// registers (an appropriate portion of EAX/EDX), passes arguments in C order, 683// and requires that the callee pop its arguments off the stack (allowing proper 684// tail calls), and has the same return value conventions as C calling convs. 685// 686// This calling convention always arranges for the callee pop value to be 8n+4 687// bytes, which is needed for tail recursion elimination and stack alignment 688// reasons. 689// 690// Note that this can be enhanced in the future to pass fp vals in registers 691// (when we have a global fp allocator) and do other tricks. 692// 693 694/// AddLiveIn - This helper function adds the specified physical register to the 695/// MachineFunction as a live in value. It also creates a corresponding virtual 696/// register for it. 697static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, 698 TargetRegisterClass *RC) { 699 assert(RC->contains(PReg) && "Not the correct regclass!"); 700 unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC); 701 MF.addLiveIn(PReg, VReg); 702 return VReg; 703} 704 705 706std::vector<SDOperand> 707X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) { 708 std::vector<SDOperand> ArgValues; 709 710 MachineFunction &MF = DAG.getMachineFunction(); 711 MachineFrameInfo *MFI = MF.getFrameInfo(); 712 713 // Add DAG nodes to load the arguments... On entry to a function the stack 714 // frame looks like this: 715 // 716 // [ESP] -- return address 717 // [ESP + 4] -- first nonreg argument (leftmost lexically) 718 // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size 719 // ... 720 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot 721 722 // Keep track of the number of integer regs passed so far. This can be either 723 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both 724 // used). 725 unsigned NumIntRegs = 0; 726 727 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) { 728 MVT::ValueType ObjectVT = getValueType(I->getType()); 729 unsigned ArgIncrement = 4; 730 unsigned ObjSize = 0; 731 SDOperand ArgValue; 732 733 switch (ObjectVT) { 734 default: assert(0 && "Unhandled argument type!"); 735 case MVT::i1: 736 case MVT::i8: 737 if (NumIntRegs < 2) { 738 if (!I->use_empty()) { 739 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL, 740 X86::R8RegisterClass); 741 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8); 742 DAG.setRoot(ArgValue.getValue(1)); 743 if (ObjectVT == MVT::i1) 744 // FIXME: Should insert a assertzext here. 745 ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue); 746 } 747 ++NumIntRegs; 748 break; 749 } 750 751 ObjSize = 1; 752 break; 753 case MVT::i16: 754 if (NumIntRegs < 2) { 755 if (!I->use_empty()) { 756 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX, 757 X86::R16RegisterClass); 758 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16); 759 DAG.setRoot(ArgValue.getValue(1)); 760 } 761 ++NumIntRegs; 762 break; 763 } 764 ObjSize = 2; 765 break; 766 case MVT::i32: 767 if (NumIntRegs < 2) { 768 if (!I->use_empty()) { 769 unsigned VReg = AddLiveIn(MF,NumIntRegs ? X86::EDX : X86::EAX, 770 X86::R32RegisterClass); 771 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32); 772 DAG.setRoot(ArgValue.getValue(1)); 773 } 774 ++NumIntRegs; 775 break; 776 } 777 ObjSize = 4; 778 break; 779 case MVT::i64: 780 if (NumIntRegs == 0) { 781 if (!I->use_empty()) { 782 unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass); 783 unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass); 784 785 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32); 786 SDOperand Hi = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32); 787 DAG.setRoot(Hi.getValue(1)); 788 789 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi); 790 } 791 NumIntRegs = 2; 792 break; 793 } else if (NumIntRegs == 1) { 794 if (!I->use_empty()) { 795 unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass); 796 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32); 797 DAG.setRoot(Low.getValue(1)); 798 799 // Load the high part from memory. 800 // Create the frame index object for this incoming parameter... 801 int FI = MFI->CreateFixedObject(4, ArgOffset); 802 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 803 SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN, 804 DAG.getSrcValue(NULL)); 805 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi); 806 } 807 ArgOffset += 4; 808 NumIntRegs = 2; 809 break; 810 } 811 ObjSize = ArgIncrement = 8; 812 break; 813 case MVT::f32: ObjSize = 4; break; 814 case MVT::f64: ObjSize = ArgIncrement = 8; break; 815 } 816 817 // Don't codegen dead arguments. FIXME: remove this check when we can nuke 818 // dead loads. 819 if (ObjSize && !I->use_empty()) { 820 // Create the frame index object for this incoming parameter... 821 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); 822 823 // Create the SelectionDAG nodes corresponding to a load from this 824 // parameter. 825 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32); 826 827 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN, 828 DAG.getSrcValue(NULL)); 829 } else if (ArgValue.Val == 0) { 830 if (MVT::isInteger(ObjectVT)) 831 ArgValue = DAG.getConstant(0, ObjectVT); 832 else 833 ArgValue = DAG.getConstantFP(0, ObjectVT); 834 } 835 ArgValues.push_back(ArgValue); 836 837 if (ObjSize) 838 ArgOffset += ArgIncrement; // Move on to the next argument. 839 } 840 841 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 842 // arguments and the arguments after the retaddr has been pushed are aligned. 843 if ((ArgOffset & 7) == 0) 844 ArgOffset += 4; 845 846 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 847 ReturnAddrIndex = 0; // No return address slot generated yet. 848 BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments. 849 BytesCallerReserves = 0; 850 851 // Finally, inform the code generator which regs we return values in. 852 switch (getValueType(F.getReturnType())) { 853 default: assert(0 && "Unknown type!"); 854 case MVT::isVoid: break; 855 case MVT::i1: 856 case MVT::i8: 857 case MVT::i16: 858 case MVT::i32: 859 MF.addLiveOut(X86::EAX); 860 break; 861 case MVT::i64: 862 MF.addLiveOut(X86::EAX); 863 MF.addLiveOut(X86::EDX); 864 break; 865 case MVT::f32: 866 case MVT::f64: 867 MF.addLiveOut(X86::ST0); 868 break; 869 } 870 return ArgValues; 871} 872 873std::pair<SDOperand, SDOperand> 874X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy, 875 bool isTailCall, SDOperand Callee, 876 ArgListTy &Args, SelectionDAG &DAG) { 877 // Count how many bytes are to be pushed on the stack. 878 unsigned NumBytes = 0; 879 880 // Keep track of the number of integer regs passed so far. This can be either 881 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both 882 // used). 883 unsigned NumIntRegs = 0; 884 885 for (unsigned i = 0, e = Args.size(); i != e; ++i) 886 switch (getValueType(Args[i].second)) { 887 default: assert(0 && "Unknown value type!"); 888 case MVT::i1: 889 case MVT::i8: 890 case MVT::i16: 891 case MVT::i32: 892 if (NumIntRegs < 2) { 893 ++NumIntRegs; 894 break; 895 } 896 // fall through 897 case MVT::f32: 898 NumBytes += 4; 899 break; 900 case MVT::i64: 901 if (NumIntRegs == 0) { 902 NumIntRegs = 2; 903 break; 904 } else if (NumIntRegs == 1) { 905 NumIntRegs = 2; 906 NumBytes += 4; 907 break; 908 } 909 910 // fall through 911 case MVT::f64: 912 NumBytes += 8; 913 break; 914 } 915 916 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 917 // arguments and the arguments after the retaddr has been pushed are aligned. 918 if ((NumBytes & 7) == 0) 919 NumBytes += 4; 920 921 Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain, 922 DAG.getConstant(NumBytes, getPointerTy())); 923 924 // Arguments go on the stack in reverse order, as specified by the ABI. 925 unsigned ArgOffset = 0; 926 SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32); 927 NumIntRegs = 0; 928 std::vector<SDOperand> Stores; 929 std::vector<SDOperand> RegValuesToPass; 930 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 931 switch (getValueType(Args[i].second)) { 932 default: assert(0 && "Unexpected ValueType for argument!"); 933 case MVT::i1: 934 Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first); 935 // Fall through. 936 case MVT::i8: 937 case MVT::i16: 938 case MVT::i32: 939 if (NumIntRegs < 2) { 940 RegValuesToPass.push_back(Args[i].first); 941 ++NumIntRegs; 942 break; 943 } 944 // Fall through 945 case MVT::f32: { 946 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 947 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 948 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 949 Args[i].first, PtrOff, 950 DAG.getSrcValue(NULL))); 951 ArgOffset += 4; 952 break; 953 } 954 case MVT::i64: 955 if (NumIntRegs < 2) { // Can pass part of it in regs? 956 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, 957 Args[i].first, DAG.getConstant(1, MVT::i32)); 958 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, 959 Args[i].first, DAG.getConstant(0, MVT::i32)); 960 RegValuesToPass.push_back(Lo); 961 ++NumIntRegs; 962 if (NumIntRegs < 2) { // Pass both parts in regs? 963 RegValuesToPass.push_back(Hi); 964 ++NumIntRegs; 965 } else { 966 // Pass the high part in memory. 967 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 968 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 969 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 970 Hi, PtrOff, DAG.getSrcValue(NULL))); 971 ArgOffset += 4; 972 } 973 break; 974 } 975 // Fall through 976 case MVT::f64: 977 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); 978 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff); 979 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 980 Args[i].first, PtrOff, 981 DAG.getSrcValue(NULL))); 982 ArgOffset += 8; 983 break; 984 } 985 } 986 if (!Stores.empty()) 987 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); 988 989 // Make sure the instruction takes 8n+4 bytes to make sure the start of the 990 // arguments and the arguments after the retaddr has been pushed are aligned. 991 if ((ArgOffset & 7) == 0) 992 ArgOffset += 4; 993 994 std::vector<MVT::ValueType> RetVals; 995 MVT::ValueType RetTyVT = getValueType(RetTy); 996 997 RetVals.push_back(MVT::Other); 998 999 // The result values produced have to be legal. Promote the result. 1000 switch (RetTyVT) { 1001 case MVT::isVoid: break; 1002 default: 1003 RetVals.push_back(RetTyVT); 1004 break; 1005 case MVT::i1: 1006 case MVT::i8: 1007 case MVT::i16: 1008 RetVals.push_back(MVT::i32); 1009 break; 1010 case MVT::f32: 1011 if (X86ScalarSSE) 1012 RetVals.push_back(MVT::f32); 1013 else 1014 RetVals.push_back(MVT::f64); 1015 break; 1016 case MVT::i64: 1017 RetVals.push_back(MVT::i32); 1018 RetVals.push_back(MVT::i32); 1019 break; 1020 } 1021 1022 if (X86DAGIsel) { 1023 // Build a sequence of copy-to-reg nodes chained together with token chain 1024 // and flag operands which copy the outgoing args into registers. 1025 SDOperand InFlag; 1026 for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) { 1027 unsigned CCReg; 1028 SDOperand RegToPass = RegValuesToPass[i]; 1029 switch (RegToPass.getValueType()) { 1030 default: assert(0 && "Bad thing to pass in regs"); 1031 case MVT::i8: 1032 CCReg = (i == 0) ? X86::AL : X86::DL; 1033 break; 1034 case MVT::i16: 1035 CCReg = (i == 0) ? X86::AX : X86::DX; 1036 break; 1037 case MVT::i32: 1038 CCReg = (i == 0) ? X86::EAX : X86::EDX; 1039 break; 1040 } 1041 1042 Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag); 1043 InFlag = Chain.getValue(1); 1044 } 1045 1046 std::vector<MVT::ValueType> NodeTys; 1047 NodeTys.push_back(MVT::Other); // Returns a chain 1048 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 1049 std::vector<SDOperand> Ops; 1050 Ops.push_back(Chain); 1051 Ops.push_back(Callee); 1052 if (InFlag.Val) 1053 Ops.push_back(InFlag); 1054 1055 // FIXME: Do not generate X86ISD::TAILCALL for now. 1056 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops); 1057 InFlag = Chain.getValue(1); 1058 1059 NodeTys.clear(); 1060 NodeTys.push_back(MVT::Other); // Returns a chain 1061 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. 1062 Ops.clear(); 1063 Ops.push_back(Chain); 1064 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1065 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1066 Ops.push_back(InFlag); 1067 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, Ops); 1068 InFlag = Chain.getValue(1); 1069 1070 SDOperand RetVal; 1071 if (RetTyVT != MVT::isVoid) { 1072 switch (RetTyVT) { 1073 default: assert(0 && "Unknown value type to return!"); 1074 case MVT::i1: 1075 case MVT::i8: 1076 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag); 1077 Chain = RetVal.getValue(1); 1078 if (RetTyVT == MVT::i1) 1079 RetVal = DAG.getNode(ISD::TRUNCATE, MVT::i1, RetVal); 1080 break; 1081 case MVT::i16: 1082 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); 1083 Chain = RetVal.getValue(1); 1084 break; 1085 case MVT::i32: 1086 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 1087 Chain = RetVal.getValue(1); 1088 break; 1089 case MVT::i64: { 1090 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag); 1091 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32, 1092 Lo.getValue(2)); 1093 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); 1094 Chain = Hi.getValue(1); 1095 break; 1096 } 1097 case MVT::f32: 1098 case MVT::f64: { 1099 std::vector<MVT::ValueType> Tys; 1100 Tys.push_back(MVT::f64); 1101 Tys.push_back(MVT::Other); 1102 Tys.push_back(MVT::Flag); 1103 std::vector<SDOperand> Ops; 1104 Ops.push_back(Chain); 1105 Ops.push_back(InFlag); 1106 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops); 1107 Chain = RetVal.getValue(1); 1108 InFlag = RetVal.getValue(2); 1109 if (X86ScalarSSE) { 1110 // FIXME:Currently the FST is flagged to the FP_GET_RESULT. This 1111 // shouldn't be necessary except for RFP cannot be live across 1112 // multiple blocks. When stackifier is fixed, they can be uncoupled. 1113 unsigned Size = MVT::getSizeInBits(MVT::f64)/8; 1114 MachineFunction &MF = DAG.getMachineFunction(); 1115 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 1116 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1117 Tys.clear(); 1118 Tys.push_back(MVT::Other); 1119 Ops.clear(); 1120 Ops.push_back(Chain); 1121 Ops.push_back(RetVal); 1122 Ops.push_back(StackSlot); 1123 Ops.push_back(DAG.getValueType(RetTyVT)); 1124 Ops.push_back(InFlag); 1125 Chain = DAG.getNode(X86ISD::FST, Tys, Ops); 1126 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot, 1127 DAG.getSrcValue(NULL)); 1128 Chain = RetVal.getValue(1); 1129 } 1130 1131 if (RetTyVT == MVT::f32 && !X86ScalarSSE) 1132 // FIXME: we would really like to remember that this FP_ROUND 1133 // operation is okay to eliminate if we allow excess FP precision. 1134 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); 1135 break; 1136 } 1137 } 1138 } 1139 1140 return std::make_pair(RetVal, Chain); 1141 } else { 1142 std::vector<SDOperand> Ops; 1143 Ops.push_back(Chain); 1144 Ops.push_back(Callee); 1145 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1146 // Callee pops all arg values on the stack. 1147 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy())); 1148 1149 // Pass register arguments as needed. 1150 Ops.insert(Ops.end(), RegValuesToPass.begin(), RegValuesToPass.end()); 1151 1152 SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, 1153 RetVals, Ops); 1154 Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall); 1155 1156 SDOperand ResultVal; 1157 switch (RetTyVT) { 1158 case MVT::isVoid: break; 1159 default: 1160 ResultVal = TheCall.getValue(1); 1161 break; 1162 case MVT::i1: 1163 case MVT::i8: 1164 case MVT::i16: 1165 ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1)); 1166 break; 1167 case MVT::f32: 1168 // FIXME: we would really like to remember that this FP_ROUND operation is 1169 // okay to eliminate if we allow excess FP precision. 1170 ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1)); 1171 break; 1172 case MVT::i64: 1173 ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1), 1174 TheCall.getValue(2)); 1175 break; 1176 } 1177 1178 return std::make_pair(ResultVal, Chain); 1179 } 1180} 1181 1182SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 1183 if (ReturnAddrIndex == 0) { 1184 // Set up a frame object for the return address. 1185 MachineFunction &MF = DAG.getMachineFunction(); 1186 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); 1187 } 1188 1189 return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32); 1190} 1191 1192 1193 1194std::pair<SDOperand, SDOperand> X86TargetLowering:: 1195LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth, 1196 SelectionDAG &DAG) { 1197 SDOperand Result; 1198 if (Depth) // Depths > 0 not supported yet! 1199 Result = DAG.getConstant(0, getPointerTy()); 1200 else { 1201 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); 1202 if (!isFrameAddress) 1203 // Just load the return address 1204 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI, 1205 DAG.getSrcValue(NULL)); 1206 else 1207 Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI, 1208 DAG.getConstant(4, MVT::i32)); 1209 } 1210 return std::make_pair(Result, Chain); 1211} 1212 1213/// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode 1214/// which corresponds to the condition code. 1215static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) { 1216 switch (X86CC) { 1217 default: assert(0 && "Unknown X86 conditional code!"); 1218 case X86ISD::COND_A: return X86::JA; 1219 case X86ISD::COND_AE: return X86::JAE; 1220 case X86ISD::COND_B: return X86::JB; 1221 case X86ISD::COND_BE: return X86::JBE; 1222 case X86ISD::COND_E: return X86::JE; 1223 case X86ISD::COND_G: return X86::JG; 1224 case X86ISD::COND_GE: return X86::JGE; 1225 case X86ISD::COND_L: return X86::JL; 1226 case X86ISD::COND_LE: return X86::JLE; 1227 case X86ISD::COND_NE: return X86::JNE; 1228 case X86ISD::COND_NO: return X86::JNO; 1229 case X86ISD::COND_NP: return X86::JNP; 1230 case X86ISD::COND_NS: return X86::JNS; 1231 case X86ISD::COND_O: return X86::JO; 1232 case X86ISD::COND_P: return X86::JP; 1233 case X86ISD::COND_S: return X86::JS; 1234 } 1235} 1236 1237/// getX86CC - do a one to one translation of a ISD::CondCode to the X86 1238/// specific condition code. It returns a X86ISD::COND_INVALID if it cannot 1239/// do a direct translation. 1240static unsigned getX86CC(SDOperand CC, bool isFP) { 1241 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 1242 unsigned X86CC = X86ISD::COND_INVALID; 1243 if (!isFP) { 1244 switch (SetCCOpcode) { 1245 default: break; 1246 case ISD::SETEQ: X86CC = X86ISD::COND_E; break; 1247 case ISD::SETGT: X86CC = X86ISD::COND_G; break; 1248 case ISD::SETGE: X86CC = X86ISD::COND_GE; break; 1249 case ISD::SETLT: X86CC = X86ISD::COND_L; break; 1250 case ISD::SETLE: X86CC = X86ISD::COND_LE; break; 1251 case ISD::SETNE: X86CC = X86ISD::COND_NE; break; 1252 case ISD::SETULT: X86CC = X86ISD::COND_B; break; 1253 case ISD::SETUGT: X86CC = X86ISD::COND_A; break; 1254 case ISD::SETULE: X86CC = X86ISD::COND_BE; break; 1255 case ISD::SETUGE: X86CC = X86ISD::COND_AE; break; 1256 } 1257 } else { 1258 // On a floating point condition, the flags are set as follows: 1259 // ZF PF CF op 1260 // 0 | 0 | 0 | X > Y 1261 // 0 | 0 | 1 | X < Y 1262 // 1 | 0 | 0 | X == Y 1263 // 1 | 1 | 1 | unordered 1264 switch (SetCCOpcode) { 1265 default: break; 1266 case ISD::SETUEQ: 1267 case ISD::SETEQ: X86CC = X86ISD::COND_E; break; 1268 case ISD::SETOGT: 1269 case ISD::SETGT: X86CC = X86ISD::COND_A; break; 1270 case ISD::SETOGE: 1271 case ISD::SETGE: X86CC = X86ISD::COND_AE; break; 1272 case ISD::SETULT: 1273 case ISD::SETLT: X86CC = X86ISD::COND_B; break; 1274 case ISD::SETULE: 1275 case ISD::SETLE: X86CC = X86ISD::COND_BE; break; 1276 case ISD::SETONE: 1277 case ISD::SETNE: X86CC = X86ISD::COND_NE; break; 1278 case ISD::SETUO: X86CC = X86ISD::COND_P; break; 1279 case ISD::SETO: X86CC = X86ISD::COND_NP; break; 1280 } 1281 } 1282 return X86CC; 1283} 1284 1285/// hasFPCMov - is there a floating point cmov for the specific X86 condition 1286/// code. Current x86 isa includes the following FP cmov instructions: 1287/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 1288static bool hasFPCMov(unsigned X86CC) { 1289 switch (X86CC) { 1290 default: 1291 return false; 1292 case X86ISD::COND_B: 1293 case X86ISD::COND_BE: 1294 case X86ISD::COND_E: 1295 case X86ISD::COND_P: 1296 case X86ISD::COND_A: 1297 case X86ISD::COND_AE: 1298 case X86ISD::COND_NE: 1299 case X86ISD::COND_NP: 1300 return true; 1301 } 1302} 1303 1304MachineBasicBlock * 1305X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI, 1306 MachineBasicBlock *BB) { 1307 switch (MI->getOpcode()) { 1308 default: assert(false && "Unexpected instr type to insert"); 1309 case X86::CMOV_FR32: 1310 case X86::CMOV_FR64: { 1311 // To "insert" a SELECT_CC instruction, we actually have to insert the diamond 1312 // control-flow pattern. The incoming instruction knows the destination vreg 1313 // to set, the condition code register to branch on, the true/false values to 1314 // select between, and a branch opcode to use. 1315 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1316 ilist<MachineBasicBlock>::iterator It = BB; 1317 ++It; 1318 1319 // thisMBB: 1320 // ... 1321 // TrueVal = ... 1322 // cmpTY ccX, r1, r2 1323 // bCC copy1MBB 1324 // fallthrough --> copy0MBB 1325 MachineBasicBlock *thisMBB = BB; 1326 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB); 1327 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB); 1328 unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue()); 1329 BuildMI(BB, Opc, 1).addMBB(sinkMBB); 1330 MachineFunction *F = BB->getParent(); 1331 F->getBasicBlockList().insert(It, copy0MBB); 1332 F->getBasicBlockList().insert(It, sinkMBB); 1333 // Update machine-CFG edges 1334 BB->addSuccessor(copy0MBB); 1335 BB->addSuccessor(sinkMBB); 1336 1337 // copy0MBB: 1338 // %FalseValue = ... 1339 // # fallthrough to sinkMBB 1340 BB = copy0MBB; 1341 1342 // Update machine-CFG edges 1343 BB->addSuccessor(sinkMBB); 1344 1345 // sinkMBB: 1346 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 1347 // ... 1348 BB = sinkMBB; 1349 BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg()) 1350 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 1351 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 1352 1353 delete MI; // The pseudo instruction is gone now. 1354 return BB; 1355 } 1356 1357 case X86::FP_TO_INT16_IN_MEM: 1358 case X86::FP_TO_INT32_IN_MEM: 1359 case X86::FP_TO_INT64_IN_MEM: { 1360 // Change the floating point control register to use "round towards zero" 1361 // mode when truncating to an integer value. 1362 MachineFunction *F = BB->getParent(); 1363 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 1364 addFrameReference(BuildMI(BB, X86::FNSTCW16m, 4), CWFrameIdx); 1365 1366 // Load the old value of the high byte of the control word... 1367 unsigned OldCW = 1368 F->getSSARegMap()->createVirtualRegister(X86::R16RegisterClass); 1369 addFrameReference(BuildMI(BB, X86::MOV16rm, 4, OldCW), CWFrameIdx); 1370 1371 // Set the high part to be round to zero... 1372 addFrameReference(BuildMI(BB, X86::MOV16mi, 5), CWFrameIdx).addImm(0xC7F); 1373 1374 // Reload the modified control word now... 1375 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx); 1376 1377 // Restore the memory image of control word to original value 1378 addFrameReference(BuildMI(BB, X86::MOV16mr, 5), CWFrameIdx).addReg(OldCW); 1379 1380 // Get the X86 opcode to use. 1381 unsigned Opc; 1382 switch (MI->getOpcode()) { 1383 case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break; 1384 case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break; 1385 case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break; 1386 } 1387 1388 X86AddressMode AM; 1389 MachineOperand &Op = MI->getOperand(0); 1390 if (Op.isRegister()) { 1391 AM.BaseType = X86AddressMode::RegBase; 1392 AM.Base.Reg = Op.getReg(); 1393 } else { 1394 AM.BaseType = X86AddressMode::FrameIndexBase; 1395 AM.Base.FrameIndex = Op.getFrameIndex(); 1396 } 1397 Op = MI->getOperand(1); 1398 if (Op.isImmediate()) 1399 AM.Scale = Op.getImmedValue(); 1400 Op = MI->getOperand(2); 1401 if (Op.isImmediate()) 1402 AM.IndexReg = Op.getImmedValue(); 1403 Op = MI->getOperand(3); 1404 if (Op.isGlobalAddress()) { 1405 AM.GV = Op.getGlobal(); 1406 } else { 1407 AM.Disp = Op.getImmedValue(); 1408 } 1409 addFullAddress(BuildMI(BB, Opc, 5), AM).addReg(MI->getOperand(4).getReg()); 1410 1411 // Reload the original control word now. 1412 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx); 1413 1414 delete MI; // The pseudo instruction is gone now. 1415 return BB; 1416 } 1417 } 1418} 1419 1420 1421//===----------------------------------------------------------------------===// 1422// X86 Custom Lowering Hooks 1423//===----------------------------------------------------------------------===// 1424 1425/// LowerOperation - Provide custom lowering hooks for some operations. 1426/// 1427SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 1428 switch (Op.getOpcode()) { 1429 default: assert(0 && "Should not custom lower this!"); 1430 case ISD::ADD_PARTS: 1431 case ISD::SUB_PARTS: { 1432 assert(Op.getNumOperands() == 4 && Op.getValueType() == MVT::i32 && 1433 "Not an i64 add/sub!"); 1434 bool isAdd = Op.getOpcode() == ISD::ADD_PARTS; 1435 std::vector<MVT::ValueType> Tys; 1436 Tys.push_back(MVT::i32); 1437 Tys.push_back(MVT::Flag); 1438 std::vector<SDOperand> Ops; 1439 Ops.push_back(Op.getOperand(0)); 1440 Ops.push_back(Op.getOperand(2)); 1441 SDOperand Lo = DAG.getNode(isAdd ? X86ISD::ADD_FLAG : X86ISD::SUB_FLAG, 1442 Tys, Ops); 1443 SDOperand Hi = DAG.getNode(isAdd ? X86ISD::ADC : X86ISD::SBB, MVT::i32, 1444 Op.getOperand(1), Op.getOperand(3), 1445 Lo.getValue(1)); 1446 Tys.clear(); 1447 Tys.push_back(MVT::i32); 1448 Tys.push_back(MVT::i32); 1449 Ops.clear(); 1450 Ops.push_back(Lo); 1451 Ops.push_back(Hi); 1452 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1453 } 1454 case ISD::SHL_PARTS: 1455 case ISD::SRA_PARTS: 1456 case ISD::SRL_PARTS: { 1457 assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 && 1458 "Not an i64 shift!"); 1459 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 1460 SDOperand ShOpLo = Op.getOperand(0); 1461 SDOperand ShOpHi = Op.getOperand(1); 1462 SDOperand ShAmt = Op.getOperand(2); 1463 SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi, 1464 DAG.getConstant(31, MVT::i8)) 1465 : DAG.getConstant(0, MVT::i32); 1466 1467 SDOperand Tmp2, Tmp3; 1468 if (Op.getOpcode() == ISD::SHL_PARTS) { 1469 Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt); 1470 Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt); 1471 } else { 1472 Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt); 1473 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, MVT::i32, ShOpHi, ShAmt); 1474 } 1475 1476 SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag, 1477 ShAmt, DAG.getConstant(32, MVT::i8)); 1478 1479 SDOperand Hi, Lo; 1480 SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1481 1482 std::vector<MVT::ValueType> Tys; 1483 Tys.push_back(MVT::i32); 1484 Tys.push_back(MVT::Flag); 1485 std::vector<SDOperand> Ops; 1486 if (Op.getOpcode() == ISD::SHL_PARTS) { 1487 Ops.push_back(Tmp2); 1488 Ops.push_back(Tmp3); 1489 Ops.push_back(CC); 1490 Ops.push_back(InFlag); 1491 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1492 InFlag = Hi.getValue(1); 1493 1494 Ops.clear(); 1495 Ops.push_back(Tmp3); 1496 Ops.push_back(Tmp1); 1497 Ops.push_back(CC); 1498 Ops.push_back(InFlag); 1499 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1500 } else { 1501 Ops.push_back(Tmp2); 1502 Ops.push_back(Tmp3); 1503 Ops.push_back(CC); 1504 Ops.push_back(InFlag); 1505 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1506 InFlag = Lo.getValue(1); 1507 1508 Ops.clear(); 1509 Ops.push_back(Tmp3); 1510 Ops.push_back(Tmp1); 1511 Ops.push_back(CC); 1512 Ops.push_back(InFlag); 1513 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops); 1514 } 1515 1516 Tys.clear(); 1517 Tys.push_back(MVT::i32); 1518 Tys.push_back(MVT::i32); 1519 Ops.clear(); 1520 Ops.push_back(Lo); 1521 Ops.push_back(Hi); 1522 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1523 } 1524 case ISD::SINT_TO_FP: { 1525 assert(Op.getValueType() == MVT::f64 && 1526 Op.getOperand(0).getValueType() <= MVT::i64 && 1527 Op.getOperand(0).getValueType() >= MVT::i16 && 1528 "Unknown SINT_TO_FP to lower!"); 1529 1530 SDOperand Result; 1531 MVT::ValueType SrcVT = Op.getOperand(0).getValueType(); 1532 unsigned Size = MVT::getSizeInBits(SrcVT)/8; 1533 MachineFunction &MF = DAG.getMachineFunction(); 1534 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 1535 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1536 SDOperand Chain = DAG.getNode(ISD::STORE, MVT::Other, 1537 DAG.getEntryNode(), Op.getOperand(0), 1538 StackSlot, DAG.getSrcValue(NULL)); 1539 1540 // Build the FILD 1541 std::vector<MVT::ValueType> Tys; 1542 Tys.push_back(MVT::f64); 1543 Tys.push_back(MVT::Flag); 1544 std::vector<SDOperand> Ops; 1545 Ops.push_back(Chain); 1546 Ops.push_back(StackSlot); 1547 Ops.push_back(DAG.getValueType(SrcVT)); 1548 Result = DAG.getNode(X86ISD::FILD, Tys, Ops); 1549 return Result; 1550 } 1551 case ISD::FP_TO_SINT: { 1552 assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 && 1553 Op.getOperand(0).getValueType() == MVT::f64 && 1554 "Unknown FP_TO_SINT to lower!"); 1555 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 1556 // stack slot. 1557 MachineFunction &MF = DAG.getMachineFunction(); 1558 unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8; 1559 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 1560 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 1561 1562 unsigned Opc; 1563 switch (Op.getValueType()) { 1564 default: assert(0 && "Invalid FP_TO_SINT to lower!"); 1565 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 1566 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 1567 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 1568 } 1569 1570 // Build the FP_TO_INT*_IN_MEM 1571 std::vector<SDOperand> Ops; 1572 Ops.push_back(DAG.getEntryNode()); 1573 Ops.push_back(Op.getOperand(0)); 1574 Ops.push_back(StackSlot); 1575 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops); 1576 1577 // Load the result. 1578 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, 1579 DAG.getSrcValue(NULL)); 1580 } 1581 case ISD::READCYCLECOUNTER: { 1582 std::vector<MVT::ValueType> Tys; 1583 Tys.push_back(MVT::Other); 1584 Tys.push_back(MVT::Flag); 1585 std::vector<SDOperand> Ops; 1586 Ops.push_back(Op.getOperand(0)); 1587 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops); 1588 Ops.clear(); 1589 Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1))); 1590 Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX, 1591 MVT::i32, Ops[0].getValue(2))); 1592 Ops.push_back(Ops[1].getValue(1)); 1593 Tys[0] = Tys[1] = MVT::i32; 1594 Tys.push_back(MVT::Other); 1595 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops); 1596 } 1597 case ISD::SETCC: { 1598 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 1599 SDOperand CC = Op.getOperand(2); 1600 SDOperand Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1601 Op.getOperand(0), Op.getOperand(1)); 1602 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 1603 bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType()); 1604 unsigned X86CC = getX86CC(CC, isFP); 1605 if (X86CC != X86ISD::COND_INVALID) { 1606 return DAG.getNode(X86ISD::SETCC, MVT::i8, 1607 DAG.getConstant(X86CC, MVT::i8), Cond); 1608 } else { 1609 assert(isFP && "Illegal integer SetCC!"); 1610 1611 std::vector<MVT::ValueType> Tys; 1612 std::vector<SDOperand> Ops; 1613 switch (SetCCOpcode) { 1614 default: assert(false && "Illegal floating point SetCC!"); 1615 case ISD::SETOEQ: { // !PF & ZF 1616 Tys.push_back(MVT::i8); 1617 Tys.push_back(MVT::Flag); 1618 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1619 Ops.push_back(Cond); 1620 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1621 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1622 DAG.getConstant(X86ISD::COND_E, MVT::i8), 1623 Tmp1.getValue(1)); 1624 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1625 } 1626 case ISD::SETOLT: { // !PF & CF 1627 Tys.push_back(MVT::i8); 1628 Tys.push_back(MVT::Flag); 1629 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1630 Ops.push_back(Cond); 1631 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1632 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1633 DAG.getConstant(X86ISD::COND_B, MVT::i8), 1634 Tmp1.getValue(1)); 1635 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1636 } 1637 case ISD::SETOLE: { // !PF & (CF || ZF) 1638 Tys.push_back(MVT::i8); 1639 Tys.push_back(MVT::Flag); 1640 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8)); 1641 Ops.push_back(Cond); 1642 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1643 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1644 DAG.getConstant(X86ISD::COND_BE, MVT::i8), 1645 Tmp1.getValue(1)); 1646 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); 1647 } 1648 case ISD::SETUGT: { // PF | (!ZF & !CF) 1649 Tys.push_back(MVT::i8); 1650 Tys.push_back(MVT::Flag); 1651 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1652 Ops.push_back(Cond); 1653 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1654 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1655 DAG.getConstant(X86ISD::COND_A, MVT::i8), 1656 Tmp1.getValue(1)); 1657 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1658 } 1659 case ISD::SETUGE: { // PF | !CF 1660 Tys.push_back(MVT::i8); 1661 Tys.push_back(MVT::Flag); 1662 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1663 Ops.push_back(Cond); 1664 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1665 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1666 DAG.getConstant(X86ISD::COND_AE, MVT::i8), 1667 Tmp1.getValue(1)); 1668 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1669 } 1670 case ISD::SETUNE: { // PF | !ZF 1671 Tys.push_back(MVT::i8); 1672 Tys.push_back(MVT::Flag); 1673 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8)); 1674 Ops.push_back(Cond); 1675 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops); 1676 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8, 1677 DAG.getConstant(X86ISD::COND_NE, MVT::i8), 1678 Tmp1.getValue(1)); 1679 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); 1680 } 1681 } 1682 } 1683 } 1684 case ISD::SELECT: { 1685 MVT::ValueType VT = Op.getValueType(); 1686 bool isFP = MVT::isFloatingPoint(VT); 1687 bool isFPStack = isFP && (X86Vector < SSE2); 1688 bool isFPSSE = isFP && (X86Vector >= SSE2); 1689 bool addTest = false; 1690 SDOperand Op0 = Op.getOperand(0); 1691 SDOperand Cond, CC; 1692 if (Op0.getOpcode() == X86ISD::SETCC) { 1693 // If condition flag is set by a X86ISD::CMP, then make a copy of it 1694 // (since flag operand cannot be shared). If the X86ISD::SETCC does not 1695 // have another use it will be eliminated. 1696 // If the X86ISD::SETCC has more than one use, then it's probably better 1697 // to use a test instead of duplicating the X86ISD::CMP (for register 1698 // pressure reason). 1699 // FIXME: Check number of live Op0 uses since we are in the middle of 1700 // legalization process. 1701 if (Op0.hasOneUse() && Op0.getOperand(1).getOpcode() == X86ISD::CMP) { 1702 CC = Op0.getOperand(0); 1703 Cond = Op0.getOperand(1); 1704 // Make a copy as flag result cannot be used by more than one. 1705 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1706 Cond.getOperand(0), Cond.getOperand(1)); 1707 addTest = 1708 isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); 1709 } else 1710 addTest = true; 1711 } else if (Op0.getOpcode() == ISD::SETCC) { 1712 CC = Op0.getOperand(2); 1713 bool isFP = MVT::isFloatingPoint(Op0.getOperand(1).getValueType()); 1714 unsigned X86CC = getX86CC(CC, isFP); 1715 CC = DAG.getConstant(X86CC, MVT::i8); 1716 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1717 Op0.getOperand(0), Op0.getOperand(1)); 1718 } else 1719 addTest = true; 1720 1721 if (addTest) { 1722 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1723 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0); 1724 } 1725 1726 std::vector<MVT::ValueType> Tys; 1727 Tys.push_back(Op.getValueType()); 1728 Tys.push_back(MVT::Flag); 1729 std::vector<SDOperand> Ops; 1730 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 1731 // condition is true. 1732 Ops.push_back(Op.getOperand(2)); 1733 Ops.push_back(Op.getOperand(1)); 1734 Ops.push_back(CC); 1735 Ops.push_back(Cond); 1736 return DAG.getNode(X86ISD::CMOV, Tys, Ops); 1737 } 1738 case ISD::BRCOND: { 1739 bool addTest = false; 1740 SDOperand Cond = Op.getOperand(1); 1741 SDOperand Dest = Op.getOperand(2); 1742 SDOperand CC; 1743 if (Cond.getOpcode() == X86ISD::SETCC) { 1744 // If condition flag is set by a X86ISD::CMP, then make a copy of it 1745 // (since flag operand cannot be shared). If the X86ISD::SETCC does not 1746 // have another use it will be eliminated. 1747 // If the X86ISD::SETCC has more than one use, then it's probably better 1748 // to use a test instead of duplicating the X86ISD::CMP (for register 1749 // pressure reason). 1750 // FIXME: Check number of live Cond uses since we are in the middle of 1751 // legalization process. 1752 if (Cond.hasOneUse() && Cond.getOperand(1).getOpcode() == X86ISD::CMP) { 1753 CC = Cond.getOperand(0); 1754 Cond = Cond.getOperand(1); 1755 // Make a copy as flag result cannot be used by more than one. 1756 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1757 Cond.getOperand(0), Cond.getOperand(1)); 1758 } else 1759 addTest = true; 1760 } else if (Cond.getOpcode() == ISD::SETCC) { 1761 CC = Cond.getOperand(2); 1762 bool isFP = MVT::isFloatingPoint(Cond.getOperand(1).getValueType()); 1763 unsigned X86CC = getX86CC(CC, isFP); 1764 CC = DAG.getConstant(X86CC, MVT::i8); 1765 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag, 1766 Cond.getOperand(0), Cond.getOperand(1)); 1767 } else 1768 addTest = true; 1769 1770 if (addTest) { 1771 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8); 1772 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond); 1773 } 1774 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), 1775 Op.getOperand(0), Op.getOperand(2), CC, Cond); 1776 } 1777 case ISD::RET: { 1778 // Can only be return void. 1779 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0), 1780 DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 1781 } 1782 case ISD::MEMSET: { 1783 SDOperand InFlag; 1784 SDOperand Chain = Op.getOperand(0); 1785 unsigned Align = 1786 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); 1787 if (Align == 0) Align = 1; 1788 1789 MVT::ValueType AVT; 1790 SDOperand Count; 1791 if (ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2))) { 1792 unsigned ValReg; 1793 unsigned Val = ValC->getValue() & 255; 1794 1795 // If the value is a constant, then we can potentially use larger sets. 1796 switch (Align & 3) { 1797 case 2: // WORD aligned 1798 AVT = MVT::i16; 1799 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1800 Count = DAG.getConstant(I->getValue() / 2, MVT::i32); 1801 else 1802 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1803 DAG.getConstant(1, MVT::i8)); 1804 Val = (Val << 8) | Val; 1805 ValReg = X86::AX; 1806 break; 1807 case 0: // DWORD aligned 1808 AVT = MVT::i32; 1809 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1810 Count = DAG.getConstant(I->getValue() / 4, MVT::i32); 1811 else 1812 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1813 DAG.getConstant(2, MVT::i8)); 1814 Val = (Val << 8) | Val; 1815 Val = (Val << 16) | Val; 1816 ValReg = X86::EAX; 1817 break; 1818 default: // Byte aligned 1819 AVT = MVT::i8; 1820 Count = Op.getOperand(3); 1821 ValReg = X86::AL; 1822 break; 1823 } 1824 1825 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), 1826 InFlag); 1827 InFlag = Chain.getValue(1); 1828 } else { 1829 AVT = MVT::i8; 1830 Count = Op.getOperand(3); 1831 Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag); 1832 InFlag = Chain.getValue(1); 1833 } 1834 1835 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag); 1836 InFlag = Chain.getValue(1); 1837 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag); 1838 InFlag = Chain.getValue(1); 1839 1840 return DAG.getNode(X86ISD::REP_STOS, MVT::Other, Chain, 1841 DAG.getValueType(AVT), InFlag); 1842 } 1843 case ISD::MEMCPY: { 1844 SDOperand Chain = Op.getOperand(0); 1845 unsigned Align = 1846 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); 1847 if (Align == 0) Align = 1; 1848 1849 MVT::ValueType AVT; 1850 SDOperand Count; 1851 switch (Align & 3) { 1852 case 2: // WORD aligned 1853 AVT = MVT::i16; 1854 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1855 Count = DAG.getConstant(I->getValue() / 2, MVT::i32); 1856 else 1857 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1858 DAG.getConstant(1, MVT::i8)); 1859 break; 1860 case 0: // DWORD aligned 1861 AVT = MVT::i32; 1862 if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3))) 1863 Count = DAG.getConstant(I->getValue() / 4, MVT::i32); 1864 else 1865 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3), 1866 DAG.getConstant(2, MVT::i8)); 1867 break; 1868 default: // Byte aligned 1869 AVT = MVT::i8; 1870 Count = Op.getOperand(3); 1871 break; 1872 } 1873 1874 SDOperand InFlag; 1875 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag); 1876 InFlag = Chain.getValue(1); 1877 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag); 1878 InFlag = Chain.getValue(1); 1879 Chain = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag); 1880 InFlag = Chain.getValue(1); 1881 1882 return DAG.getNode(X86ISD::REP_MOVS, MVT::Other, Chain, 1883 DAG.getValueType(AVT), InFlag); 1884 } 1885 case ISD::GlobalAddress: { 1886 SDOperand Result; 1887 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 1888 // For Darwin, external and weak symbols are indirect, so we want to load 1889 // the value at address GV, not the value of GV itself. This means that 1890 // the GlobalAddress must be in the base or index register of the address, 1891 // not the GV offset field. 1892 if (getTargetMachine(). 1893 getSubtarget<X86Subtarget>().getIndirectExternAndWeakGlobals() && 1894 (GV->hasWeakLinkage() || GV->isExternal())) 1895 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), 1896 DAG.getTargetGlobalAddress(GV, getPointerTy()), 1897 DAG.getSrcValue(NULL)); 1898 return Result; 1899 } 1900 } 1901} 1902 1903const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 1904 switch (Opcode) { 1905 default: return NULL; 1906 case X86ISD::ADD_FLAG: return "X86ISD::ADD_FLAG"; 1907 case X86ISD::SUB_FLAG: return "X86ISD::SUB_FLAG"; 1908 case X86ISD::ADC: return "X86ISD::ADC"; 1909 case X86ISD::SBB: return "X86ISD::SBB"; 1910 case X86ISD::SHLD: return "X86ISD::SHLD"; 1911 case X86ISD::SHRD: return "X86ISD::SHRD"; 1912 case X86ISD::FILD: return "X86ISD::FILD"; 1913 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 1914 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 1915 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 1916 case X86ISD::FLD: return "X86ISD::FLD"; 1917 case X86ISD::FST: return "X86ISD::FST"; 1918 case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT"; 1919 case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT"; 1920 case X86ISD::CALL: return "X86ISD::CALL"; 1921 case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; 1922 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 1923 case X86ISD::CMP: return "X86ISD::CMP"; 1924 case X86ISD::TEST: return "X86ISD::TEST"; 1925 case X86ISD::SETCC: return "X86ISD::SETCC"; 1926 case X86ISD::CMOV: return "X86ISD::CMOV"; 1927 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 1928 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 1929 case X86ISD::REP_STOS: return "X86ISD::RET_STOS"; 1930 case X86ISD::REP_MOVS: return "X86ISD::RET_MOVS"; 1931 } 1932} 1933 1934bool X86TargetLowering::isMaskedValueZeroForTargetNode(const SDOperand &Op, 1935 uint64_t Mask) const { 1936 1937 unsigned Opc = Op.getOpcode(); 1938 1939 switch (Opc) { 1940 default: 1941 assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node"); 1942 break; 1943 case X86ISD::SETCC: return (Mask & 1) == 0; 1944 } 1945 1946 return false; 1947} 1948