PPCISelLowering.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f
1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the PPCISelLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "PPCISelLowering.h" 15#include "MCTargetDesc/PPCPredicates.h" 16#include "PPCMachineFunctionInfo.h" 17#include "PPCPerfectShuffle.h" 18#include "PPCTargetMachine.h" 19#include "PPCTargetObjectFile.h" 20#include "llvm/ADT/STLExtras.h" 21#include "llvm/ADT/StringSwitch.h" 22#include "llvm/CodeGen/CallingConvLower.h" 23#include "llvm/CodeGen/MachineFrameInfo.h" 24#include "llvm/CodeGen/MachineFunction.h" 25#include "llvm/CodeGen/MachineInstrBuilder.h" 26#include "llvm/CodeGen/MachineRegisterInfo.h" 27#include "llvm/CodeGen/SelectionDAG.h" 28#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 29#include "llvm/IR/CallingConv.h" 30#include "llvm/IR/Constants.h" 31#include "llvm/IR/DerivedTypes.h" 32#include "llvm/IR/Function.h" 33#include "llvm/IR/Intrinsics.h" 34#include "llvm/Support/CommandLine.h" 35#include "llvm/Support/ErrorHandling.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/raw_ostream.h" 38#include "llvm/Target/TargetOptions.h" 39using namespace llvm; 40 41static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc", 42cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); 43 44static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref", 45cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); 46 47static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned", 48cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); 49 50// FIXME: Remove this once the bug has been fixed! 51extern cl::opt<bool> ANDIGlueBug; 52 53static TargetLoweringObjectFile *CreateTLOF(const PPCTargetMachine &TM) { 54 if (TM.getSubtargetImpl()->isDarwin()) 55 return new TargetLoweringObjectFileMachO(); 56 57 if (TM.getSubtargetImpl()->isSVR4ABI()) 58 return new PPC64LinuxTargetObjectFile(); 59 60 return new TargetLoweringObjectFileELF(); 61} 62 63PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM) 64 : TargetLowering(TM, CreateTLOF(TM)), PPCSubTarget(*TM.getSubtargetImpl()) { 65 const PPCSubtarget *Subtarget = &TM.getSubtarget<PPCSubtarget>(); 66 67 setPow2DivIsCheap(); 68 69 // Use _setjmp/_longjmp instead of setjmp/longjmp. 70 setUseUnderscoreSetJmp(true); 71 setUseUnderscoreLongJmp(true); 72 73 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all 74 // arguments are at least 4/8 bytes aligned. 75 bool isPPC64 = Subtarget->isPPC64(); 76 setMinStackArgumentAlignment(isPPC64 ? 8:4); 77 78 // Set up the register classes. 79 addRegisterClass(MVT::i32, &PPC::GPRCRegClass); 80 addRegisterClass(MVT::f32, &PPC::F4RCRegClass); 81 addRegisterClass(MVT::f64, &PPC::F8RCRegClass); 82 83 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD 84 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 85 setLoadExtAction(ISD::SEXTLOAD, MVT::i8, Expand); 86 87 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 88 89 // PowerPC has pre-inc load and store's. 90 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); 91 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); 92 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); 93 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); 94 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); 95 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); 96 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); 97 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); 98 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); 99 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); 100 101 if (Subtarget->useCRBits()) { 102 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 103 104 if (isPPC64 || Subtarget->hasFPCVT()) { 105 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); 106 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, 107 isPPC64 ? MVT::i64 : MVT::i32); 108 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); 109 AddPromotedToType (ISD::UINT_TO_FP, MVT::i1, 110 isPPC64 ? MVT::i64 : MVT::i32); 111 } else { 112 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); 113 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); 114 } 115 116 // PowerPC does not support direct load / store of condition registers 117 setOperationAction(ISD::LOAD, MVT::i1, Custom); 118 setOperationAction(ISD::STORE, MVT::i1, Custom); 119 120 // FIXME: Remove this once the ANDI glue bug is fixed: 121 if (ANDIGlueBug) 122 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 123 124 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 125 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 126 setTruncStoreAction(MVT::i64, MVT::i1, Expand); 127 setTruncStoreAction(MVT::i32, MVT::i1, Expand); 128 setTruncStoreAction(MVT::i16, MVT::i1, Expand); 129 setTruncStoreAction(MVT::i8, MVT::i1, Expand); 130 131 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); 132 } 133 134 // This is used in the ppcf128->int sequence. Note it has different semantics 135 // from FP_ROUND: that rounds to nearest, this rounds to zero. 136 setOperationAction(ISD::FP_ROUND_INREG, MVT::ppcf128, Custom); 137 138 // We do not currently implement these libm ops for PowerPC. 139 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); 140 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); 141 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); 142 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); 143 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); 144 setOperationAction(ISD::FREM, MVT::ppcf128, Expand); 145 146 // PowerPC has no SREM/UREM instructions 147 setOperationAction(ISD::SREM, MVT::i32, Expand); 148 setOperationAction(ISD::UREM, MVT::i32, Expand); 149 setOperationAction(ISD::SREM, MVT::i64, Expand); 150 setOperationAction(ISD::UREM, MVT::i64, Expand); 151 152 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. 153 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 154 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 155 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 156 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 157 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 158 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 159 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 160 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 161 162 // We don't support sin/cos/sqrt/fmod/pow 163 setOperationAction(ISD::FSIN , MVT::f64, Expand); 164 setOperationAction(ISD::FCOS , MVT::f64, Expand); 165 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 166 setOperationAction(ISD::FREM , MVT::f64, Expand); 167 setOperationAction(ISD::FPOW , MVT::f64, Expand); 168 setOperationAction(ISD::FMA , MVT::f64, Legal); 169 setOperationAction(ISD::FSIN , MVT::f32, Expand); 170 setOperationAction(ISD::FCOS , MVT::f32, Expand); 171 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 172 setOperationAction(ISD::FREM , MVT::f32, Expand); 173 setOperationAction(ISD::FPOW , MVT::f32, Expand); 174 setOperationAction(ISD::FMA , MVT::f32, Legal); 175 176 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 177 178 // If we're enabling GP optimizations, use hardware square root 179 if (!Subtarget->hasFSQRT() && 180 !(TM.Options.UnsafeFPMath && 181 Subtarget->hasFRSQRTE() && Subtarget->hasFRE())) 182 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 183 184 if (!Subtarget->hasFSQRT() && 185 !(TM.Options.UnsafeFPMath && 186 Subtarget->hasFRSQRTES() && Subtarget->hasFRES())) 187 setOperationAction(ISD::FSQRT, MVT::f32, Expand); 188 189 if (Subtarget->hasFCPSGN()) { 190 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); 191 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); 192 } else { 193 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 194 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 195 } 196 197 if (Subtarget->hasFPRND()) { 198 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 199 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 200 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 201 setOperationAction(ISD::FROUND, MVT::f64, Legal); 202 203 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 204 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 205 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 206 setOperationAction(ISD::FROUND, MVT::f32, Legal); 207 } 208 209 // PowerPC does not have BSWAP, CTPOP or CTTZ 210 setOperationAction(ISD::BSWAP, MVT::i32 , Expand); 211 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 212 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); 213 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); 214 setOperationAction(ISD::BSWAP, MVT::i64 , Expand); 215 setOperationAction(ISD::CTTZ , MVT::i64 , Expand); 216 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); 217 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); 218 219 if (Subtarget->hasPOPCNTD()) { 220 setOperationAction(ISD::CTPOP, MVT::i32 , Legal); 221 setOperationAction(ISD::CTPOP, MVT::i64 , Legal); 222 } else { 223 setOperationAction(ISD::CTPOP, MVT::i32 , Expand); 224 setOperationAction(ISD::CTPOP, MVT::i64 , Expand); 225 } 226 227 // PowerPC does not have ROTR 228 setOperationAction(ISD::ROTR, MVT::i32 , Expand); 229 setOperationAction(ISD::ROTR, MVT::i64 , Expand); 230 231 if (!Subtarget->useCRBits()) { 232 // PowerPC does not have Select 233 setOperationAction(ISD::SELECT, MVT::i32, Expand); 234 setOperationAction(ISD::SELECT, MVT::i64, Expand); 235 setOperationAction(ISD::SELECT, MVT::f32, Expand); 236 setOperationAction(ISD::SELECT, MVT::f64, Expand); 237 } 238 239 // PowerPC wants to turn select_cc of FP into fsel when possible. 240 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 241 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 242 243 // PowerPC wants to optimize integer setcc a bit 244 if (!Subtarget->useCRBits()) 245 setOperationAction(ISD::SETCC, MVT::i32, Custom); 246 247 // PowerPC does not have BRCOND which requires SetCC 248 if (!Subtarget->useCRBits()) 249 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 250 251 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 252 253 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. 254 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 255 256 // PowerPC does not have [U|S]INT_TO_FP 257 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); 258 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); 259 260 setOperationAction(ISD::BITCAST, MVT::f32, Expand); 261 setOperationAction(ISD::BITCAST, MVT::i32, Expand); 262 setOperationAction(ISD::BITCAST, MVT::i64, Expand); 263 setOperationAction(ISD::BITCAST, MVT::f64, Expand); 264 265 // We cannot sextinreg(i1). Expand to shifts. 266 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 267 268 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 269 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 270 // support continuation, user-level threading, and etc.. As a result, no 271 // other SjLj exception interfaces are implemented and please don't build 272 // your own exception handling based on them. 273 // LLVM/Clang supports zero-cost DWARF exception handling. 274 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 275 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 276 277 // We want to legalize GlobalAddress and ConstantPool nodes into the 278 // appropriate instructions to materialize the address. 279 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 280 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 281 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 282 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 283 setOperationAction(ISD::JumpTable, MVT::i32, Custom); 284 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 285 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 286 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 287 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 288 setOperationAction(ISD::JumpTable, MVT::i64, Custom); 289 290 // TRAP is legal. 291 setOperationAction(ISD::TRAP, MVT::Other, Legal); 292 293 // TRAMPOLINE is custom lowered. 294 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 295 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 296 297 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 298 setOperationAction(ISD::VASTART , MVT::Other, Custom); 299 300 if (Subtarget->isSVR4ABI()) { 301 if (isPPC64) { 302 // VAARG always uses double-word chunks, so promote anything smaller. 303 setOperationAction(ISD::VAARG, MVT::i1, Promote); 304 AddPromotedToType (ISD::VAARG, MVT::i1, MVT::i64); 305 setOperationAction(ISD::VAARG, MVT::i8, Promote); 306 AddPromotedToType (ISD::VAARG, MVT::i8, MVT::i64); 307 setOperationAction(ISD::VAARG, MVT::i16, Promote); 308 AddPromotedToType (ISD::VAARG, MVT::i16, MVT::i64); 309 setOperationAction(ISD::VAARG, MVT::i32, Promote); 310 AddPromotedToType (ISD::VAARG, MVT::i32, MVT::i64); 311 setOperationAction(ISD::VAARG, MVT::Other, Expand); 312 } else { 313 // VAARG is custom lowered with the 32-bit SVR4 ABI. 314 setOperationAction(ISD::VAARG, MVT::Other, Custom); 315 setOperationAction(ISD::VAARG, MVT::i64, Custom); 316 } 317 } else 318 setOperationAction(ISD::VAARG, MVT::Other, Expand); 319 320 if (Subtarget->isSVR4ABI() && !isPPC64) 321 // VACOPY is custom lowered with the 32-bit SVR4 ABI. 322 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 323 else 324 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 325 326 // Use the default implementation. 327 setOperationAction(ISD::VAEND , MVT::Other, Expand); 328 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); 329 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); 330 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); 331 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); 332 333 // We want to custom lower some of our intrinsics. 334 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 335 336 // To handle counter-based loop conditions. 337 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); 338 339 // Comparisons that require checking two conditions. 340 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 341 setCondCodeAction(ISD::SETULT, MVT::f64, Expand); 342 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 343 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); 344 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 345 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); 346 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); 347 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); 348 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 349 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); 350 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 351 setCondCodeAction(ISD::SETONE, MVT::f64, Expand); 352 353 if (Subtarget->has64BitSupport()) { 354 // They also have instructions for converting between i64 and fp. 355 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 356 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); 357 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 358 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 359 // This is just the low 32 bits of a (signed) fp->i64 conversion. 360 // We cannot do this with Promote because i64 is not a legal type. 361 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 362 363 if (PPCSubTarget.hasLFIWAX() || Subtarget->isPPC64()) 364 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 365 } else { 366 // PowerPC does not have FP_TO_UINT on 32-bit implementations. 367 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); 368 } 369 370 // With the instructions enabled under FPCVT, we can do everything. 371 if (PPCSubTarget.hasFPCVT()) { 372 if (Subtarget->has64BitSupport()) { 373 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 374 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 375 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 376 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 377 } 378 379 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 380 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 381 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 382 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 383 } 384 385 if (Subtarget->use64BitRegs()) { 386 // 64-bit PowerPC implementations can support i64 types directly 387 addRegisterClass(MVT::i64, &PPC::G8RCRegClass); 388 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or 389 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 390 // 64-bit PowerPC wants to expand i128 shifts itself. 391 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 392 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 393 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 394 } else { 395 // 32-bit PowerPC wants to expand i64 shifts itself. 396 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 397 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 398 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 399 } 400 401 if (Subtarget->hasAltivec()) { 402 // First set operation action for all vector types to expand. Then we 403 // will selectively turn on ones that can be effectively codegen'd. 404 for (unsigned i = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 405 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { 406 MVT::SimpleValueType VT = (MVT::SimpleValueType)i; 407 408 // add/sub are legal for all supported vector VT's. 409 setOperationAction(ISD::ADD , VT, Legal); 410 setOperationAction(ISD::SUB , VT, Legal); 411 412 // We promote all shuffles to v16i8. 413 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); 414 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); 415 416 // We promote all non-typed operations to v4i32. 417 setOperationAction(ISD::AND , VT, Promote); 418 AddPromotedToType (ISD::AND , VT, MVT::v4i32); 419 setOperationAction(ISD::OR , VT, Promote); 420 AddPromotedToType (ISD::OR , VT, MVT::v4i32); 421 setOperationAction(ISD::XOR , VT, Promote); 422 AddPromotedToType (ISD::XOR , VT, MVT::v4i32); 423 setOperationAction(ISD::LOAD , VT, Promote); 424 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); 425 setOperationAction(ISD::SELECT, VT, Promote); 426 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); 427 setOperationAction(ISD::STORE, VT, Promote); 428 AddPromotedToType (ISD::STORE, VT, MVT::v4i32); 429 430 // No other operations are legal. 431 setOperationAction(ISD::MUL , VT, Expand); 432 setOperationAction(ISD::SDIV, VT, Expand); 433 setOperationAction(ISD::SREM, VT, Expand); 434 setOperationAction(ISD::UDIV, VT, Expand); 435 setOperationAction(ISD::UREM, VT, Expand); 436 setOperationAction(ISD::FDIV, VT, Expand); 437 setOperationAction(ISD::FREM, VT, Expand); 438 setOperationAction(ISD::FNEG, VT, Expand); 439 setOperationAction(ISD::FSQRT, VT, Expand); 440 setOperationAction(ISD::FLOG, VT, Expand); 441 setOperationAction(ISD::FLOG10, VT, Expand); 442 setOperationAction(ISD::FLOG2, VT, Expand); 443 setOperationAction(ISD::FEXP, VT, Expand); 444 setOperationAction(ISD::FEXP2, VT, Expand); 445 setOperationAction(ISD::FSIN, VT, Expand); 446 setOperationAction(ISD::FCOS, VT, Expand); 447 setOperationAction(ISD::FABS, VT, Expand); 448 setOperationAction(ISD::FPOWI, VT, Expand); 449 setOperationAction(ISD::FFLOOR, VT, Expand); 450 setOperationAction(ISD::FCEIL, VT, Expand); 451 setOperationAction(ISD::FTRUNC, VT, Expand); 452 setOperationAction(ISD::FRINT, VT, Expand); 453 setOperationAction(ISD::FNEARBYINT, VT, Expand); 454 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); 455 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 456 setOperationAction(ISD::BUILD_VECTOR, VT, Expand); 457 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 458 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 459 setOperationAction(ISD::UDIVREM, VT, Expand); 460 setOperationAction(ISD::SDIVREM, VT, Expand); 461 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); 462 setOperationAction(ISD::FPOW, VT, Expand); 463 setOperationAction(ISD::BSWAP, VT, Expand); 464 setOperationAction(ISD::CTPOP, VT, Expand); 465 setOperationAction(ISD::CTLZ, VT, Expand); 466 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 467 setOperationAction(ISD::CTTZ, VT, Expand); 468 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 469 setOperationAction(ISD::VSELECT, VT, Expand); 470 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 471 472 for (unsigned j = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 473 j <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++j) { 474 MVT::SimpleValueType InnerVT = (MVT::SimpleValueType)j; 475 setTruncStoreAction(VT, InnerVT, Expand); 476 } 477 setLoadExtAction(ISD::SEXTLOAD, VT, Expand); 478 setLoadExtAction(ISD::ZEXTLOAD, VT, Expand); 479 setLoadExtAction(ISD::EXTLOAD, VT, Expand); 480 } 481 482 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle 483 // with merges, splats, etc. 484 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); 485 486 setOperationAction(ISD::AND , MVT::v4i32, Legal); 487 setOperationAction(ISD::OR , MVT::v4i32, Legal); 488 setOperationAction(ISD::XOR , MVT::v4i32, Legal); 489 setOperationAction(ISD::LOAD , MVT::v4i32, Legal); 490 setOperationAction(ISD::SELECT, MVT::v4i32, 491 Subtarget->useCRBits() ? Legal : Expand); 492 setOperationAction(ISD::STORE , MVT::v4i32, Legal); 493 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 494 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 495 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 496 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 497 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 498 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 499 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 500 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 501 502 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); 503 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); 504 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); 505 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); 506 507 setOperationAction(ISD::MUL, MVT::v4f32, Legal); 508 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 509 510 if (TM.Options.UnsafeFPMath || Subtarget->hasVSX()) { 511 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 512 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 513 } 514 515 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 516 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 517 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 518 519 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); 520 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); 521 522 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); 523 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); 524 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); 525 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 526 527 // Altivec does not contain unordered floating-point compare instructions 528 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); 529 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); 530 setCondCodeAction(ISD::SETUGT, MVT::v4f32, Expand); 531 setCondCodeAction(ISD::SETUGE, MVT::v4f32, Expand); 532 setCondCodeAction(ISD::SETULT, MVT::v4f32, Expand); 533 setCondCodeAction(ISD::SETULE, MVT::v4f32, Expand); 534 535 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); 536 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); 537 538 if (Subtarget->hasVSX()) { 539 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 540 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 541 542 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 543 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 544 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 545 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 546 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 547 548 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 549 550 setOperationAction(ISD::MUL, MVT::v2f64, Legal); 551 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 552 553 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 554 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 555 556 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); 557 setOperationAction(ISD::VSELECT, MVT::v8i16, Legal); 558 setOperationAction(ISD::VSELECT, MVT::v4i32, Legal); 559 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 560 setOperationAction(ISD::VSELECT, MVT::v2f64, Legal); 561 562 // Share the Altivec comparison restrictions. 563 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); 564 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); 565 setCondCodeAction(ISD::SETUGT, MVT::v2f64, Expand); 566 setCondCodeAction(ISD::SETUGE, MVT::v2f64, Expand); 567 setCondCodeAction(ISD::SETULT, MVT::v2f64, Expand); 568 setCondCodeAction(ISD::SETULE, MVT::v2f64, Expand); 569 570 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); 571 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); 572 573 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 574 setOperationAction(ISD::STORE, MVT::v2f64, Legal); 575 576 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); 577 578 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); 579 580 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); 581 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); 582 583 // VSX v2i64 only supports non-arithmetic operations. 584 setOperationAction(ISD::ADD, MVT::v2i64, Expand); 585 setOperationAction(ISD::SUB, MVT::v2i64, Expand); 586 587 setOperationAction(ISD::SHL, MVT::v2i64, Expand); 588 setOperationAction(ISD::SRA, MVT::v2i64, Expand); 589 setOperationAction(ISD::SRL, MVT::v2i64, Expand); 590 591 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 592 593 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 594 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); 595 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 596 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); 597 598 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); 599 600 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 601 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 602 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 603 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 604 605 // Vector operation legalization checks the result type of 606 // SIGN_EXTEND_INREG, overall legalization checks the inner type. 607 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); 608 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); 609 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); 610 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); 611 612 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); 613 } 614 } 615 616 if (Subtarget->has64BitSupport()) { 617 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); 618 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); 619 } 620 621 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Expand); 622 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Expand); 623 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); 624 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); 625 626 setBooleanContents(ZeroOrOneBooleanContent); 627 // Altivec instructions set fields to all zeros or all ones. 628 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 629 630 if (isPPC64) { 631 setStackPointerRegisterToSaveRestore(PPC::X1); 632 setExceptionPointerRegister(PPC::X3); 633 setExceptionSelectorRegister(PPC::X4); 634 } else { 635 setStackPointerRegisterToSaveRestore(PPC::R1); 636 setExceptionPointerRegister(PPC::R3); 637 setExceptionSelectorRegister(PPC::R4); 638 } 639 640 // We have target-specific dag combine patterns for the following nodes: 641 setTargetDAGCombine(ISD::SINT_TO_FP); 642 setTargetDAGCombine(ISD::LOAD); 643 setTargetDAGCombine(ISD::STORE); 644 setTargetDAGCombine(ISD::BR_CC); 645 if (Subtarget->useCRBits()) 646 setTargetDAGCombine(ISD::BRCOND); 647 setTargetDAGCombine(ISD::BSWAP); 648 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 649 650 setTargetDAGCombine(ISD::SIGN_EXTEND); 651 setTargetDAGCombine(ISD::ZERO_EXTEND); 652 setTargetDAGCombine(ISD::ANY_EXTEND); 653 654 if (Subtarget->useCRBits()) { 655 setTargetDAGCombine(ISD::TRUNCATE); 656 setTargetDAGCombine(ISD::SETCC); 657 setTargetDAGCombine(ISD::SELECT_CC); 658 } 659 660 // Use reciprocal estimates. 661 if (TM.Options.UnsafeFPMath) { 662 setTargetDAGCombine(ISD::FDIV); 663 setTargetDAGCombine(ISD::FSQRT); 664 } 665 666 // Darwin long double math library functions have $LDBL128 appended. 667 if (Subtarget->isDarwin()) { 668 setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128"); 669 setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128"); 670 setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128"); 671 setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128"); 672 setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128"); 673 setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128"); 674 setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128"); 675 setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128"); 676 setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128"); 677 setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128"); 678 } 679 680 // With 32 condition bits, we don't need to sink (and duplicate) compares 681 // aggressively in CodeGenPrep. 682 if (Subtarget->useCRBits()) 683 setHasMultipleConditionRegisters(); 684 685 setMinFunctionAlignment(2); 686 if (PPCSubTarget.isDarwin()) 687 setPrefFunctionAlignment(4); 688 689 if (isPPC64 && Subtarget->isJITCodeModel()) 690 // Temporary workaround for the inability of PPC64 JIT to handle jump 691 // tables. 692 setSupportJumpTables(false); 693 694 setInsertFencesForAtomic(true); 695 696 if (Subtarget->enableMachineScheduler()) 697 setSchedulingPreference(Sched::Source); 698 else 699 setSchedulingPreference(Sched::Hybrid); 700 701 computeRegisterProperties(); 702 703 // The Freescale cores does better with aggressive inlining of memcpy and 704 // friends. Gcc uses same threshold of 128 bytes (= 32 word stores). 705 if (Subtarget->getDarwinDirective() == PPC::DIR_E500mc || 706 Subtarget->getDarwinDirective() == PPC::DIR_E5500) { 707 MaxStoresPerMemset = 32; 708 MaxStoresPerMemsetOptSize = 16; 709 MaxStoresPerMemcpy = 32; 710 MaxStoresPerMemcpyOptSize = 8; 711 MaxStoresPerMemmove = 32; 712 MaxStoresPerMemmoveOptSize = 8; 713 714 setPrefFunctionAlignment(4); 715 } 716} 717 718/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 719/// the desired ByVal argument alignment. 720static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, 721 unsigned MaxMaxAlign) { 722 if (MaxAlign == MaxMaxAlign) 723 return; 724 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 725 if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) 726 MaxAlign = 32; 727 else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) 728 MaxAlign = 16; 729 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 730 unsigned EltAlign = 0; 731 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); 732 if (EltAlign > MaxAlign) 733 MaxAlign = EltAlign; 734 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 735 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 736 unsigned EltAlign = 0; 737 getMaxByValAlign(STy->getElementType(i), EltAlign, MaxMaxAlign); 738 if (EltAlign > MaxAlign) 739 MaxAlign = EltAlign; 740 if (MaxAlign == MaxMaxAlign) 741 break; 742 } 743 } 744} 745 746/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 747/// function arguments in the caller parameter area. 748unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty) const { 749 // Darwin passes everything on 4 byte boundary. 750 if (PPCSubTarget.isDarwin()) 751 return 4; 752 753 // 16byte and wider vectors are passed on 16byte boundary. 754 // The rest is 8 on PPC64 and 4 on PPC32 boundary. 755 unsigned Align = PPCSubTarget.isPPC64() ? 8 : 4; 756 if (PPCSubTarget.hasAltivec() || PPCSubTarget.hasQPX()) 757 getMaxByValAlign(Ty, Align, PPCSubTarget.hasQPX() ? 32 : 16); 758 return Align; 759} 760 761const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { 762 switch (Opcode) { 763 default: return nullptr; 764 case PPCISD::FSEL: return "PPCISD::FSEL"; 765 case PPCISD::FCFID: return "PPCISD::FCFID"; 766 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; 767 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; 768 case PPCISD::FRE: return "PPCISD::FRE"; 769 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; 770 case PPCISD::STFIWX: return "PPCISD::STFIWX"; 771 case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; 772 case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; 773 case PPCISD::VPERM: return "PPCISD::VPERM"; 774 case PPCISD::Hi: return "PPCISD::Hi"; 775 case PPCISD::Lo: return "PPCISD::Lo"; 776 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; 777 case PPCISD::TOC_RESTORE: return "PPCISD::TOC_RESTORE"; 778 case PPCISD::LOAD: return "PPCISD::LOAD"; 779 case PPCISD::LOAD_TOC: return "PPCISD::LOAD_TOC"; 780 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; 781 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; 782 case PPCISD::SRL: return "PPCISD::SRL"; 783 case PPCISD::SRA: return "PPCISD::SRA"; 784 case PPCISD::SHL: return "PPCISD::SHL"; 785 case PPCISD::CALL: return "PPCISD::CALL"; 786 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; 787 case PPCISD::MTCTR: return "PPCISD::MTCTR"; 788 case PPCISD::BCTRL: return "PPCISD::BCTRL"; 789 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; 790 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; 791 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; 792 case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; 793 case PPCISD::VCMP: return "PPCISD::VCMP"; 794 case PPCISD::VCMPo: return "PPCISD::VCMPo"; 795 case PPCISD::LBRX: return "PPCISD::LBRX"; 796 case PPCISD::STBRX: return "PPCISD::STBRX"; 797 case PPCISD::LARX: return "PPCISD::LARX"; 798 case PPCISD::STCX: return "PPCISD::STCX"; 799 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; 800 case PPCISD::BDNZ: return "PPCISD::BDNZ"; 801 case PPCISD::BDZ: return "PPCISD::BDZ"; 802 case PPCISD::MFFS: return "PPCISD::MFFS"; 803 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; 804 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; 805 case PPCISD::CR6SET: return "PPCISD::CR6SET"; 806 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; 807 case PPCISD::ADDIS_TOC_HA: return "PPCISD::ADDIS_TOC_HA"; 808 case PPCISD::LD_TOC_L: return "PPCISD::LD_TOC_L"; 809 case PPCISD::ADDI_TOC_L: return "PPCISD::ADDI_TOC_L"; 810 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; 811 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; 812 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; 813 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; 814 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; 815 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; 816 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; 817 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; 818 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; 819 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; 820 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; 821 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; 822 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; 823 case PPCISD::SC: return "PPCISD::SC"; 824 } 825} 826 827EVT PPCTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { 828 if (!VT.isVector()) 829 return PPCSubTarget.useCRBits() ? MVT::i1 : MVT::i32; 830 return VT.changeVectorElementTypeToInteger(); 831} 832 833//===----------------------------------------------------------------------===// 834// Node matching predicates, for use by the tblgen matching code. 835//===----------------------------------------------------------------------===// 836 837/// isFloatingPointZero - Return true if this is 0.0 or -0.0. 838static bool isFloatingPointZero(SDValue Op) { 839 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 840 return CFP->getValueAPF().isZero(); 841 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 842 // Maybe this has already been legalized into the constant pool? 843 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1))) 844 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 845 return CFP->getValueAPF().isZero(); 846 } 847 return false; 848} 849 850/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return 851/// true if Op is undef or if it matches the specified value. 852static bool isConstantOrUndef(int Op, int Val) { 853 return Op < 0 || Op == Val; 854} 855 856/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a 857/// VPKUHUM instruction. 858bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary) { 859 if (!isUnary) { 860 for (unsigned i = 0; i != 16; ++i) 861 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) 862 return false; 863 } else { 864 for (unsigned i = 0; i != 8; ++i) 865 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1) || 866 !isConstantOrUndef(N->getMaskElt(i+8), i*2+1)) 867 return false; 868 } 869 return true; 870} 871 872/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a 873/// VPKUWUM instruction. 874bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary) { 875 if (!isUnary) { 876 for (unsigned i = 0; i != 16; i += 2) 877 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 878 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) 879 return false; 880 } else { 881 for (unsigned i = 0; i != 8; i += 2) 882 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 883 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3) || 884 !isConstantOrUndef(N->getMaskElt(i+8), i*2+2) || 885 !isConstantOrUndef(N->getMaskElt(i+9), i*2+3)) 886 return false; 887 } 888 return true; 889} 890 891/// isVMerge - Common function, used to match vmrg* shuffles. 892/// 893static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, 894 unsigned LHSStart, unsigned RHSStart) { 895 if (N->getValueType(0) != MVT::v16i8) 896 return false; 897 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && 898 "Unsupported merge size!"); 899 900 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units 901 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit 902 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), 903 LHSStart+j+i*UnitSize) || 904 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), 905 RHSStart+j+i*UnitSize)) 906 return false; 907 } 908 return true; 909} 910 911/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for 912/// a VRGL* instruction with the specified unit size (1,2 or 4 bytes). 913bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 914 bool isUnary) { 915 if (!isUnary) 916 return isVMerge(N, UnitSize, 8, 24); 917 return isVMerge(N, UnitSize, 8, 8); 918} 919 920/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for 921/// a VRGH* instruction with the specified unit size (1,2 or 4 bytes). 922bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 923 bool isUnary) { 924 if (!isUnary) 925 return isVMerge(N, UnitSize, 0, 16); 926 return isVMerge(N, UnitSize, 0, 0); 927} 928 929 930/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift 931/// amount, otherwise return -1. 932int PPC::isVSLDOIShuffleMask(SDNode *N, bool isUnary) { 933 if (N->getValueType(0) != MVT::v16i8) 934 return -1; 935 936 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 937 938 // Find the first non-undef value in the shuffle mask. 939 unsigned i; 940 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) 941 /*search*/; 942 943 if (i == 16) return -1; // all undef. 944 945 // Otherwise, check to see if the rest of the elements are consecutively 946 // numbered from this value. 947 unsigned ShiftAmt = SVOp->getMaskElt(i); 948 if (ShiftAmt < i) return -1; 949 ShiftAmt -= i; 950 951 if (!isUnary) { 952 // Check the rest of the elements to see if they are consecutive. 953 for (++i; i != 16; ++i) 954 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 955 return -1; 956 } else { 957 // Check the rest of the elements to see if they are consecutive. 958 for (++i; i != 16; ++i) 959 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) 960 return -1; 961 } 962 return ShiftAmt; 963} 964 965/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand 966/// specifies a splat of a single element that is suitable for input to 967/// VSPLTB/VSPLTH/VSPLTW. 968bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { 969 assert(N->getValueType(0) == MVT::v16i8 && 970 (EltSize == 1 || EltSize == 2 || EltSize == 4)); 971 972 // This is a splat operation if each element of the permute is the same, and 973 // if the value doesn't reference the second vector. 974 unsigned ElementBase = N->getMaskElt(0); 975 976 // FIXME: Handle UNDEF elements too! 977 if (ElementBase >= 16) 978 return false; 979 980 // Check that the indices are consecutive, in the case of a multi-byte element 981 // splatted with a v16i8 mask. 982 for (unsigned i = 1; i != EltSize; ++i) 983 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) 984 return false; 985 986 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { 987 if (N->getMaskElt(i) < 0) continue; 988 for (unsigned j = 0; j != EltSize; ++j) 989 if (N->getMaskElt(i+j) != N->getMaskElt(j)) 990 return false; 991 } 992 return true; 993} 994 995/// isAllNegativeZeroVector - Returns true if all elements of build_vector 996/// are -0.0. 997bool PPC::isAllNegativeZeroVector(SDNode *N) { 998 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(N); 999 1000 APInt APVal, APUndef; 1001 unsigned BitSize; 1002 bool HasAnyUndefs; 1003 1004 if (BV->isConstantSplat(APVal, APUndef, BitSize, HasAnyUndefs, 32, true)) 1005 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 1006 return CFP->getValueAPF().isNegZero(); 1007 1008 return false; 1009} 1010 1011/// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the 1012/// specified isSplatShuffleMask VECTOR_SHUFFLE mask. 1013unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize) { 1014 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1015 assert(isSplatShuffleMask(SVOp, EltSize)); 1016 return SVOp->getMaskElt(0) / EltSize; 1017} 1018 1019/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed 1020/// by using a vspltis[bhw] instruction of the specified element size, return 1021/// the constant being splatted. The ByteSize field indicates the number of 1022/// bytes of each element [124] -> [bhw]. 1023SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { 1024 SDValue OpVal(nullptr, 0); 1025 1026 // If ByteSize of the splat is bigger than the element size of the 1027 // build_vector, then we have a case where we are checking for a splat where 1028 // multiple elements of the buildvector are folded together into a single 1029 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). 1030 unsigned EltSize = 16/N->getNumOperands(); 1031 if (EltSize < ByteSize) { 1032 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. 1033 SDValue UniquedVals[4]; 1034 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); 1035 1036 // See if all of the elements in the buildvector agree across. 1037 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 1038 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1039 // If the element isn't a constant, bail fully out. 1040 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue(); 1041 1042 1043 if (!UniquedVals[i&(Multiple-1)].getNode()) 1044 UniquedVals[i&(Multiple-1)] = N->getOperand(i); 1045 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) 1046 return SDValue(); // no match. 1047 } 1048 1049 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains 1050 // either constant or undef values that are identical for each chunk. See 1051 // if these chunks can form into a larger vspltis*. 1052 1053 // Check to see if all of the leading entries are either 0 or -1. If 1054 // neither, then this won't fit into the immediate field. 1055 bool LeadingZero = true; 1056 bool LeadingOnes = true; 1057 for (unsigned i = 0; i != Multiple-1; ++i) { 1058 if (!UniquedVals[i].getNode()) continue; // Must have been undefs. 1059 1060 LeadingZero &= cast<ConstantSDNode>(UniquedVals[i])->isNullValue(); 1061 LeadingOnes &= cast<ConstantSDNode>(UniquedVals[i])->isAllOnesValue(); 1062 } 1063 // Finally, check the least significant entry. 1064 if (LeadingZero) { 1065 if (!UniquedVals[Multiple-1].getNode()) 1066 return DAG.getTargetConstant(0, MVT::i32); // 0,0,0,undef 1067 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue(); 1068 if (Val < 16) 1069 return DAG.getTargetConstant(Val, MVT::i32); // 0,0,0,4 -> vspltisw(4) 1070 } 1071 if (LeadingOnes) { 1072 if (!UniquedVals[Multiple-1].getNode()) 1073 return DAG.getTargetConstant(~0U, MVT::i32); // -1,-1,-1,undef 1074 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue(); 1075 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) 1076 return DAG.getTargetConstant(Val, MVT::i32); 1077 } 1078 1079 return SDValue(); 1080 } 1081 1082 // Check to see if this buildvec has a single non-undef value in its elements. 1083 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 1084 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1085 if (!OpVal.getNode()) 1086 OpVal = N->getOperand(i); 1087 else if (OpVal != N->getOperand(i)) 1088 return SDValue(); 1089 } 1090 1091 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. 1092 1093 unsigned ValSizeInBytes = EltSize; 1094 uint64_t Value = 0; 1095 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) { 1096 Value = CN->getZExtValue(); 1097 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) { 1098 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); 1099 Value = FloatToBits(CN->getValueAPF().convertToFloat()); 1100 } 1101 1102 // If the splat value is larger than the element value, then we can never do 1103 // this splat. The only case that we could fit the replicated bits into our 1104 // immediate field for would be zero, and we prefer to use vxor for it. 1105 if (ValSizeInBytes < ByteSize) return SDValue(); 1106 1107 // If the element value is larger than the splat value, cut it in half and 1108 // check to see if the two halves are equal. Continue doing this until we 1109 // get to ByteSize. This allows us to handle 0x01010101 as 0x01. 1110 while (ValSizeInBytes > ByteSize) { 1111 ValSizeInBytes >>= 1; 1112 1113 // If the top half equals the bottom half, we're still ok. 1114 if (((Value >> (ValSizeInBytes*8)) & ((1 << (8*ValSizeInBytes))-1)) != 1115 (Value & ((1 << (8*ValSizeInBytes))-1))) 1116 return SDValue(); 1117 } 1118 1119 // Properly sign extend the value. 1120 int MaskVal = SignExtend32(Value, ByteSize * 8); 1121 1122 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. 1123 if (MaskVal == 0) return SDValue(); 1124 1125 // Finally, if this value fits in a 5 bit sext field, return it 1126 if (SignExtend32<5>(MaskVal) == MaskVal) 1127 return DAG.getTargetConstant(MaskVal, MVT::i32); 1128 return SDValue(); 1129} 1130 1131//===----------------------------------------------------------------------===// 1132// Addressing Mode Selection 1133//===----------------------------------------------------------------------===// 1134 1135/// isIntS16Immediate - This method tests to see if the node is either a 32-bit 1136/// or 64-bit immediate, and if the value can be accurately represented as a 1137/// sign extension from a 16-bit value. If so, this returns true and the 1138/// immediate. 1139static bool isIntS16Immediate(SDNode *N, short &Imm) { 1140 if (!isa<ConstantSDNode>(N)) 1141 return false; 1142 1143 Imm = (short)cast<ConstantSDNode>(N)->getZExtValue(); 1144 if (N->getValueType(0) == MVT::i32) 1145 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); 1146 else 1147 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 1148} 1149static bool isIntS16Immediate(SDValue Op, short &Imm) { 1150 return isIntS16Immediate(Op.getNode(), Imm); 1151} 1152 1153 1154/// SelectAddressRegReg - Given the specified addressed, check to see if it 1155/// can be represented as an indexed [r+r] operation. Returns false if it 1156/// can be more efficiently represented with [r+imm]. 1157bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, 1158 SDValue &Index, 1159 SelectionDAG &DAG) const { 1160 short imm = 0; 1161 if (N.getOpcode() == ISD::ADD) { 1162 if (isIntS16Immediate(N.getOperand(1), imm)) 1163 return false; // r+i 1164 if (N.getOperand(1).getOpcode() == PPCISD::Lo) 1165 return false; // r+i 1166 1167 Base = N.getOperand(0); 1168 Index = N.getOperand(1); 1169 return true; 1170 } else if (N.getOpcode() == ISD::OR) { 1171 if (isIntS16Immediate(N.getOperand(1), imm)) 1172 return false; // r+i can fold it if we can. 1173 1174 // If this is an or of disjoint bitfields, we can codegen this as an add 1175 // (for better address arithmetic) if the LHS and RHS of the OR are provably 1176 // disjoint. 1177 APInt LHSKnownZero, LHSKnownOne; 1178 APInt RHSKnownZero, RHSKnownOne; 1179 DAG.computeKnownBits(N.getOperand(0), 1180 LHSKnownZero, LHSKnownOne); 1181 1182 if (LHSKnownZero.getBoolValue()) { 1183 DAG.computeKnownBits(N.getOperand(1), 1184 RHSKnownZero, RHSKnownOne); 1185 // If all of the bits are known zero on the LHS or RHS, the add won't 1186 // carry. 1187 if (~(LHSKnownZero | RHSKnownZero) == 0) { 1188 Base = N.getOperand(0); 1189 Index = N.getOperand(1); 1190 return true; 1191 } 1192 } 1193 } 1194 1195 return false; 1196} 1197 1198// If we happen to be doing an i64 load or store into a stack slot that has 1199// less than a 4-byte alignment, then the frame-index elimination may need to 1200// use an indexed load or store instruction (because the offset may not be a 1201// multiple of 4). The extra register needed to hold the offset comes from the 1202// register scavenger, and it is possible that the scavenger will need to use 1203// an emergency spill slot. As a result, we need to make sure that a spill slot 1204// is allocated when doing an i64 load/store into a less-than-4-byte-aligned 1205// stack slot. 1206static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { 1207 // FIXME: This does not handle the LWA case. 1208 if (VT != MVT::i64) 1209 return; 1210 1211 // NOTE: We'll exclude negative FIs here, which come from argument 1212 // lowering, because there are no known test cases triggering this problem 1213 // using packed structures (or similar). We can remove this exclusion if 1214 // we find such a test case. The reason why this is so test-case driven is 1215 // because this entire 'fixup' is only to prevent crashes (from the 1216 // register scavenger) on not-really-valid inputs. For example, if we have: 1217 // %a = alloca i1 1218 // %b = bitcast i1* %a to i64* 1219 // store i64* a, i64 b 1220 // then the store should really be marked as 'align 1', but is not. If it 1221 // were marked as 'align 1' then the indexed form would have been 1222 // instruction-selected initially, and the problem this 'fixup' is preventing 1223 // won't happen regardless. 1224 if (FrameIdx < 0) 1225 return; 1226 1227 MachineFunction &MF = DAG.getMachineFunction(); 1228 MachineFrameInfo *MFI = MF.getFrameInfo(); 1229 1230 unsigned Align = MFI->getObjectAlignment(FrameIdx); 1231 if (Align >= 4) 1232 return; 1233 1234 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 1235 FuncInfo->setHasNonRISpills(); 1236} 1237 1238/// Returns true if the address N can be represented by a base register plus 1239/// a signed 16-bit displacement [r+imm], and if it is not better 1240/// represented as reg+reg. If Aligned is true, only accept displacements 1241/// suitable for STD and friends, i.e. multiples of 4. 1242bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, 1243 SDValue &Base, 1244 SelectionDAG &DAG, 1245 bool Aligned) const { 1246 // FIXME dl should come from parent load or store, not from address 1247 SDLoc dl(N); 1248 // If this can be more profitably realized as r+r, fail. 1249 if (SelectAddressRegReg(N, Disp, Base, DAG)) 1250 return false; 1251 1252 if (N.getOpcode() == ISD::ADD) { 1253 short imm = 0; 1254 if (isIntS16Immediate(N.getOperand(1), imm) && 1255 (!Aligned || (imm & 3) == 0)) { 1256 Disp = DAG.getTargetConstant(imm, N.getValueType()); 1257 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 1258 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 1259 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 1260 } else { 1261 Base = N.getOperand(0); 1262 } 1263 return true; // [r+i] 1264 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { 1265 // Match LOAD (ADD (X, Lo(G))). 1266 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() 1267 && "Cannot handle constant offsets yet!"); 1268 Disp = N.getOperand(1).getOperand(0); // The global address. 1269 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 1270 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 1271 Disp.getOpcode() == ISD::TargetConstantPool || 1272 Disp.getOpcode() == ISD::TargetJumpTable); 1273 Base = N.getOperand(0); 1274 return true; // [&g+r] 1275 } 1276 } else if (N.getOpcode() == ISD::OR) { 1277 short imm = 0; 1278 if (isIntS16Immediate(N.getOperand(1), imm) && 1279 (!Aligned || (imm & 3) == 0)) { 1280 // If this is an or of disjoint bitfields, we can codegen this as an add 1281 // (for better address arithmetic) if the LHS and RHS of the OR are 1282 // provably disjoint. 1283 APInt LHSKnownZero, LHSKnownOne; 1284 DAG.computeKnownBits(N.getOperand(0), LHSKnownZero, LHSKnownOne); 1285 1286 if ((LHSKnownZero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { 1287 // If all of the bits are known zero on the LHS or RHS, the add won't 1288 // carry. 1289 Base = N.getOperand(0); 1290 Disp = DAG.getTargetConstant(imm, N.getValueType()); 1291 return true; 1292 } 1293 } 1294 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 1295 // Loading from a constant address. 1296 1297 // If this address fits entirely in a 16-bit sext immediate field, codegen 1298 // this as "d, 0" 1299 short Imm; 1300 if (isIntS16Immediate(CN, Imm) && (!Aligned || (Imm & 3) == 0)) { 1301 Disp = DAG.getTargetConstant(Imm, CN->getValueType(0)); 1302 Base = DAG.getRegister(PPCSubTarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 1303 CN->getValueType(0)); 1304 return true; 1305 } 1306 1307 // Handle 32-bit sext immediates with LIS + addr mode. 1308 if ((CN->getValueType(0) == MVT::i32 || 1309 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && 1310 (!Aligned || (CN->getZExtValue() & 3) == 0)) { 1311 int Addr = (int)CN->getZExtValue(); 1312 1313 // Otherwise, break this down into an LIS + disp. 1314 Disp = DAG.getTargetConstant((short)Addr, MVT::i32); 1315 1316 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, MVT::i32); 1317 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; 1318 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); 1319 return true; 1320 } 1321 } 1322 1323 Disp = DAG.getTargetConstant(0, getPointerTy()); 1324 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 1325 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 1326 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 1327 } else 1328 Base = N; 1329 return true; // [r+0] 1330} 1331 1332/// SelectAddressRegRegOnly - Given the specified addressed, force it to be 1333/// represented as an indexed [r+r] operation. 1334bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, 1335 SDValue &Index, 1336 SelectionDAG &DAG) const { 1337 // Check to see if we can easily represent this as an [r+r] address. This 1338 // will fail if it thinks that the address is more profitably represented as 1339 // reg+imm, e.g. where imm = 0. 1340 if (SelectAddressRegReg(N, Base, Index, DAG)) 1341 return true; 1342 1343 // If the operand is an addition, always emit this as [r+r], since this is 1344 // better (for code size, and execution, as the memop does the add for free) 1345 // than emitting an explicit add. 1346 if (N.getOpcode() == ISD::ADD) { 1347 Base = N.getOperand(0); 1348 Index = N.getOperand(1); 1349 return true; 1350 } 1351 1352 // Otherwise, do it the hard way, using R0 as the base register. 1353 Base = DAG.getRegister(PPCSubTarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 1354 N.getValueType()); 1355 Index = N; 1356 return true; 1357} 1358 1359/// getPreIndexedAddressParts - returns true by value, base pointer and 1360/// offset pointer and addressing mode by reference if the node's address 1361/// can be legally represented as pre-indexed load / store address. 1362bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 1363 SDValue &Offset, 1364 ISD::MemIndexedMode &AM, 1365 SelectionDAG &DAG) const { 1366 if (DisablePPCPreinc) return false; 1367 1368 bool isLoad = true; 1369 SDValue Ptr; 1370 EVT VT; 1371 unsigned Alignment; 1372 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 1373 Ptr = LD->getBasePtr(); 1374 VT = LD->getMemoryVT(); 1375 Alignment = LD->getAlignment(); 1376 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 1377 Ptr = ST->getBasePtr(); 1378 VT = ST->getMemoryVT(); 1379 Alignment = ST->getAlignment(); 1380 isLoad = false; 1381 } else 1382 return false; 1383 1384 // PowerPC doesn't have preinc load/store instructions for vectors. 1385 if (VT.isVector()) 1386 return false; 1387 1388 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { 1389 1390 // Common code will reject creating a pre-inc form if the base pointer 1391 // is a frame index, or if N is a store and the base pointer is either 1392 // the same as or a predecessor of the value being stored. Check for 1393 // those situations here, and try with swapped Base/Offset instead. 1394 bool Swap = false; 1395 1396 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base)) 1397 Swap = true; 1398 else if (!isLoad) { 1399 SDValue Val = cast<StoreSDNode>(N)->getValue(); 1400 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) 1401 Swap = true; 1402 } 1403 1404 if (Swap) 1405 std::swap(Base, Offset); 1406 1407 AM = ISD::PRE_INC; 1408 return true; 1409 } 1410 1411 // LDU/STU can only handle immediates that are a multiple of 4. 1412 if (VT != MVT::i64) { 1413 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, false)) 1414 return false; 1415 } else { 1416 // LDU/STU need an address with at least 4-byte alignment. 1417 if (Alignment < 4) 1418 return false; 1419 1420 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, true)) 1421 return false; 1422 } 1423 1424 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 1425 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of 1426 // sext i32 to i64 when addr mode is r+i. 1427 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && 1428 LD->getExtensionType() == ISD::SEXTLOAD && 1429 isa<ConstantSDNode>(Offset)) 1430 return false; 1431 } 1432 1433 AM = ISD::PRE_INC; 1434 return true; 1435} 1436 1437//===----------------------------------------------------------------------===// 1438// LowerOperation implementation 1439//===----------------------------------------------------------------------===// 1440 1441/// GetLabelAccessInfo - Return true if we should reference labels using a 1442/// PICBase, set the HiOpFlags and LoOpFlags to the target MO flags. 1443static bool GetLabelAccessInfo(const TargetMachine &TM, unsigned &HiOpFlags, 1444 unsigned &LoOpFlags, 1445 const GlobalValue *GV = nullptr) { 1446 HiOpFlags = PPCII::MO_HA; 1447 LoOpFlags = PPCII::MO_LO; 1448 1449 // Don't use the pic base if not in PIC relocation model. Or if we are on a 1450 // non-darwin platform. We don't support PIC on other platforms yet. 1451 bool isPIC = TM.getRelocationModel() == Reloc::PIC_ && 1452 TM.getSubtarget<PPCSubtarget>().isDarwin(); 1453 if (isPIC) { 1454 HiOpFlags |= PPCII::MO_PIC_FLAG; 1455 LoOpFlags |= PPCII::MO_PIC_FLAG; 1456 } 1457 1458 // If this is a reference to a global value that requires a non-lazy-ptr, make 1459 // sure that instruction lowering adds it. 1460 if (GV && TM.getSubtarget<PPCSubtarget>().hasLazyResolverStub(GV, TM)) { 1461 HiOpFlags |= PPCII::MO_NLP_FLAG; 1462 LoOpFlags |= PPCII::MO_NLP_FLAG; 1463 1464 if (GV->hasHiddenVisibility()) { 1465 HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 1466 LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 1467 } 1468 } 1469 1470 return isPIC; 1471} 1472 1473static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, 1474 SelectionDAG &DAG) { 1475 EVT PtrVT = HiPart.getValueType(); 1476 SDValue Zero = DAG.getConstant(0, PtrVT); 1477 SDLoc DL(HiPart); 1478 1479 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); 1480 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); 1481 1482 // With PIC, the first instruction is actually "GR+hi(&G)". 1483 if (isPIC) 1484 Hi = DAG.getNode(ISD::ADD, DL, PtrVT, 1485 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); 1486 1487 // Generate non-pic code that has direct accesses to the constant pool. 1488 // The address of the global is just (hi(&g)+lo(&g)). 1489 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); 1490} 1491 1492SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, 1493 SelectionDAG &DAG) const { 1494 EVT PtrVT = Op.getValueType(); 1495 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 1496 const Constant *C = CP->getConstVal(); 1497 1498 // 64-bit SVR4 ABI code is always position-independent. 1499 // The actual address of the GlobalValue is stored in the TOC. 1500 if (PPCSubTarget.isSVR4ABI() && PPCSubTarget.isPPC64()) { 1501 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); 1502 return DAG.getNode(PPCISD::TOC_ENTRY, SDLoc(CP), MVT::i64, GA, 1503 DAG.getRegister(PPC::X2, MVT::i64)); 1504 } 1505 1506 unsigned MOHiFlag, MOLoFlag; 1507 bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag); 1508 SDValue CPIHi = 1509 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); 1510 SDValue CPILo = 1511 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); 1512 return LowerLabelRef(CPIHi, CPILo, isPIC, DAG); 1513} 1514 1515SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 1516 EVT PtrVT = Op.getValueType(); 1517 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 1518 1519 // 64-bit SVR4 ABI code is always position-independent. 1520 // The actual address of the GlobalValue is stored in the TOC. 1521 if (PPCSubTarget.isSVR4ABI() && PPCSubTarget.isPPC64()) { 1522 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1523 return DAG.getNode(PPCISD::TOC_ENTRY, SDLoc(JT), MVT::i64, GA, 1524 DAG.getRegister(PPC::X2, MVT::i64)); 1525 } 1526 1527 unsigned MOHiFlag, MOLoFlag; 1528 bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag); 1529 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); 1530 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); 1531 return LowerLabelRef(JTIHi, JTILo, isPIC, DAG); 1532} 1533 1534SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, 1535 SelectionDAG &DAG) const { 1536 EVT PtrVT = Op.getValueType(); 1537 1538 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 1539 1540 unsigned MOHiFlag, MOLoFlag; 1541 bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag); 1542 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); 1543 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); 1544 return LowerLabelRef(TgtBAHi, TgtBALo, isPIC, DAG); 1545} 1546 1547SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, 1548 SelectionDAG &DAG) const { 1549 1550 // FIXME: TLS addresses currently use medium model code sequences, 1551 // which is the most useful form. Eventually support for small and 1552 // large models could be added if users need it, at the cost of 1553 // additional complexity. 1554 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 1555 SDLoc dl(GA); 1556 const GlobalValue *GV = GA->getGlobal(); 1557 EVT PtrVT = getPointerTy(); 1558 bool is64bit = PPCSubTarget.isPPC64(); 1559 1560 TLSModel::Model Model = getTargetMachine().getTLSModel(GV); 1561 1562 if (Model == TLSModel::LocalExec) { 1563 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 1564 PPCII::MO_TPREL_HA); 1565 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 1566 PPCII::MO_TPREL_LO); 1567 SDValue TLSReg = DAG.getRegister(is64bit ? PPC::X13 : PPC::R2, 1568 is64bit ? MVT::i64 : MVT::i32); 1569 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); 1570 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); 1571 } 1572 1573 if (Model == TLSModel::InitialExec) { 1574 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 1575 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 1576 PPCII::MO_TLS); 1577 SDValue GOTPtr; 1578 if (is64bit) { 1579 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 1580 GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, 1581 PtrVT, GOTReg, TGA); 1582 } else 1583 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); 1584 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, 1585 PtrVT, TGA, GOTPtr); 1586 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); 1587 } 1588 1589 if (Model == TLSModel::GeneralDynamic) { 1590 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 1591 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 1592 SDValue GOTEntryHi = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, 1593 GOTReg, TGA); 1594 SDValue GOTEntry = DAG.getNode(PPCISD::ADDI_TLSGD_L, dl, PtrVT, 1595 GOTEntryHi, TGA); 1596 1597 // We need a chain node, and don't have one handy. The underlying 1598 // call has no side effects, so using the function entry node 1599 // suffices. 1600 SDValue Chain = DAG.getEntryNode(); 1601 Chain = DAG.getCopyToReg(Chain, dl, PPC::X3, GOTEntry); 1602 SDValue ParmReg = DAG.getRegister(PPC::X3, MVT::i64); 1603 SDValue TLSAddr = DAG.getNode(PPCISD::GET_TLS_ADDR, dl, 1604 PtrVT, ParmReg, TGA); 1605 // The return value from GET_TLS_ADDR really is in X3 already, but 1606 // some hacks are needed here to tie everything together. The extra 1607 // copies dissolve during subsequent transforms. 1608 Chain = DAG.getCopyToReg(Chain, dl, PPC::X3, TLSAddr); 1609 return DAG.getCopyFromReg(Chain, dl, PPC::X3, PtrVT); 1610 } 1611 1612 if (Model == TLSModel::LocalDynamic) { 1613 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 1614 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 1615 SDValue GOTEntryHi = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, 1616 GOTReg, TGA); 1617 SDValue GOTEntry = DAG.getNode(PPCISD::ADDI_TLSLD_L, dl, PtrVT, 1618 GOTEntryHi, TGA); 1619 1620 // We need a chain node, and don't have one handy. The underlying 1621 // call has no side effects, so using the function entry node 1622 // suffices. 1623 SDValue Chain = DAG.getEntryNode(); 1624 Chain = DAG.getCopyToReg(Chain, dl, PPC::X3, GOTEntry); 1625 SDValue ParmReg = DAG.getRegister(PPC::X3, MVT::i64); 1626 SDValue TLSAddr = DAG.getNode(PPCISD::GET_TLSLD_ADDR, dl, 1627 PtrVT, ParmReg, TGA); 1628 // The return value from GET_TLSLD_ADDR really is in X3 already, but 1629 // some hacks are needed here to tie everything together. The extra 1630 // copies dissolve during subsequent transforms. 1631 Chain = DAG.getCopyToReg(Chain, dl, PPC::X3, TLSAddr); 1632 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, PtrVT, 1633 Chain, ParmReg, TGA); 1634 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); 1635 } 1636 1637 llvm_unreachable("Unknown TLS model!"); 1638} 1639 1640SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, 1641 SelectionDAG &DAG) const { 1642 EVT PtrVT = Op.getValueType(); 1643 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op); 1644 SDLoc DL(GSDN); 1645 const GlobalValue *GV = GSDN->getGlobal(); 1646 1647 // 64-bit SVR4 ABI code is always position-independent. 1648 // The actual address of the GlobalValue is stored in the TOC. 1649 if (PPCSubTarget.isSVR4ABI() && PPCSubTarget.isPPC64()) { 1650 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); 1651 return DAG.getNode(PPCISD::TOC_ENTRY, DL, MVT::i64, GA, 1652 DAG.getRegister(PPC::X2, MVT::i64)); 1653 } 1654 1655 unsigned MOHiFlag, MOLoFlag; 1656 bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag, GV); 1657 1658 SDValue GAHi = 1659 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); 1660 SDValue GALo = 1661 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); 1662 1663 SDValue Ptr = LowerLabelRef(GAHi, GALo, isPIC, DAG); 1664 1665 // If the global reference is actually to a non-lazy-pointer, we have to do an 1666 // extra load to get the address of the global. 1667 if (MOHiFlag & PPCII::MO_NLP_FLAG) 1668 Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo(), 1669 false, false, false, 0); 1670 return Ptr; 1671} 1672 1673SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 1674 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 1675 SDLoc dl(Op); 1676 1677 if (Op.getValueType() == MVT::v2i64) { 1678 // When the operands themselves are v2i64 values, we need to do something 1679 // special because VSX has no underlying comparison operations for these. 1680 if (Op.getOperand(0).getValueType() == MVT::v2i64) { 1681 // Equality can be handled by casting to the legal type for Altivec 1682 // comparisons, everything else needs to be expanded. 1683 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 1684 return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 1685 DAG.getSetCC(dl, MVT::v4i32, 1686 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), 1687 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), 1688 CC)); 1689 } 1690 1691 return SDValue(); 1692 } 1693 1694 // We handle most of these in the usual way. 1695 return Op; 1696 } 1697 1698 // If we're comparing for equality to zero, expose the fact that this is 1699 // implented as a ctlz/srl pair on ppc, so that the dag combiner can 1700 // fold the new nodes. 1701 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1702 if (C->isNullValue() && CC == ISD::SETEQ) { 1703 EVT VT = Op.getOperand(0).getValueType(); 1704 SDValue Zext = Op.getOperand(0); 1705 if (VT.bitsLT(MVT::i32)) { 1706 VT = MVT::i32; 1707 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); 1708 } 1709 unsigned Log2b = Log2_32(VT.getSizeInBits()); 1710 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); 1711 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, 1712 DAG.getConstant(Log2b, MVT::i32)); 1713 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); 1714 } 1715 // Leave comparisons against 0 and -1 alone for now, since they're usually 1716 // optimized. FIXME: revisit this when we can custom lower all setcc 1717 // optimizations. 1718 if (C->isAllOnesValue() || C->isNullValue()) 1719 return SDValue(); 1720 } 1721 1722 // If we have an integer seteq/setne, turn it into a compare against zero 1723 // by xor'ing the rhs with the lhs, which is faster than setting a 1724 // condition register, reading it back out, and masking the correct bit. The 1725 // normal approach here uses sub to do this instead of xor. Using xor exposes 1726 // the result to other bit-twiddling opportunities. 1727 EVT LHSVT = Op.getOperand(0).getValueType(); 1728 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 1729 EVT VT = Op.getValueType(); 1730 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), 1731 Op.getOperand(1)); 1732 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, LHSVT), CC); 1733 } 1734 return SDValue(); 1735} 1736 1737SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG, 1738 const PPCSubtarget &Subtarget) const { 1739 SDNode *Node = Op.getNode(); 1740 EVT VT = Node->getValueType(0); 1741 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 1742 SDValue InChain = Node->getOperand(0); 1743 SDValue VAListPtr = Node->getOperand(1); 1744 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 1745 SDLoc dl(Node); 1746 1747 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); 1748 1749 // gpr_index 1750 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 1751 VAListPtr, MachinePointerInfo(SV), MVT::i8, 1752 false, false, 0); 1753 InChain = GprIndex.getValue(1); 1754 1755 if (VT == MVT::i64) { 1756 // Check if GprIndex is even 1757 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, 1758 DAG.getConstant(1, MVT::i32)); 1759 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, 1760 DAG.getConstant(0, MVT::i32), ISD::SETNE); 1761 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, 1762 DAG.getConstant(1, MVT::i32)); 1763 // Align GprIndex to be even if it isn't 1764 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, 1765 GprIndex); 1766 } 1767 1768 // fpr index is 1 byte after gpr 1769 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 1770 DAG.getConstant(1, MVT::i32)); 1771 1772 // fpr 1773 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 1774 FprPtr, MachinePointerInfo(SV), MVT::i8, 1775 false, false, 0); 1776 InChain = FprIndex.getValue(1); 1777 1778 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 1779 DAG.getConstant(8, MVT::i32)); 1780 1781 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 1782 DAG.getConstant(4, MVT::i32)); 1783 1784 // areas 1785 SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, 1786 MachinePointerInfo(), false, false, 1787 false, 0); 1788 InChain = OverflowArea.getValue(1); 1789 1790 SDValue RegSaveArea = DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, 1791 MachinePointerInfo(), false, false, 1792 false, 0); 1793 InChain = RegSaveArea.getValue(1); 1794 1795 // select overflow_area if index > 8 1796 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, 1797 DAG.getConstant(8, MVT::i32), ISD::SETLT); 1798 1799 // adjustment constant gpr_index * 4/8 1800 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, 1801 VT.isInteger() ? GprIndex : FprIndex, 1802 DAG.getConstant(VT.isInteger() ? 4 : 8, 1803 MVT::i32)); 1804 1805 // OurReg = RegSaveArea + RegConstant 1806 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, 1807 RegConstant); 1808 1809 // Floating types are 32 bytes into RegSaveArea 1810 if (VT.isFloatingPoint()) 1811 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, 1812 DAG.getConstant(32, MVT::i32)); 1813 1814 // increase {f,g}pr_index by 1 (or 2 if VT is i64) 1815 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, 1816 VT.isInteger() ? GprIndex : FprIndex, 1817 DAG.getConstant(VT == MVT::i64 ? 2 : 1, 1818 MVT::i32)); 1819 1820 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, 1821 VT.isInteger() ? VAListPtr : FprPtr, 1822 MachinePointerInfo(SV), 1823 MVT::i8, false, false, 0); 1824 1825 // determine if we should load from reg_save_area or overflow_area 1826 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); 1827 1828 // increase overflow_area by 4/8 if gpr/fpr > 8 1829 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, 1830 DAG.getConstant(VT.isInteger() ? 4 : 8, 1831 MVT::i32)); 1832 1833 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, 1834 OverflowAreaPlusN); 1835 1836 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, 1837 OverflowAreaPtr, 1838 MachinePointerInfo(), 1839 MVT::i32, false, false, 0); 1840 1841 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(), 1842 false, false, false, 0); 1843} 1844 1845SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG, 1846 const PPCSubtarget &Subtarget) const { 1847 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); 1848 1849 // We have to copy the entire va_list struct: 1850 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte 1851 return DAG.getMemcpy(Op.getOperand(0), Op, 1852 Op.getOperand(1), Op.getOperand(2), 1853 DAG.getConstant(12, MVT::i32), 8, false, true, 1854 MachinePointerInfo(), MachinePointerInfo()); 1855} 1856 1857SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 1858 SelectionDAG &DAG) const { 1859 return Op.getOperand(0); 1860} 1861 1862SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 1863 SelectionDAG &DAG) const { 1864 SDValue Chain = Op.getOperand(0); 1865 SDValue Trmp = Op.getOperand(1); // trampoline 1866 SDValue FPtr = Op.getOperand(2); // nested function 1867 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 1868 SDLoc dl(Op); 1869 1870 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 1871 bool isPPC64 = (PtrVT == MVT::i64); 1872 Type *IntPtrTy = 1873 DAG.getTargetLoweringInfo().getDataLayout()->getIntPtrType( 1874 *DAG.getContext()); 1875 1876 TargetLowering::ArgListTy Args; 1877 TargetLowering::ArgListEntry Entry; 1878 1879 Entry.Ty = IntPtrTy; 1880 Entry.Node = Trmp; Args.push_back(Entry); 1881 1882 // TrampSize == (isPPC64 ? 48 : 40); 1883 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, 1884 isPPC64 ? MVT::i64 : MVT::i32); 1885 Args.push_back(Entry); 1886 1887 Entry.Node = FPtr; Args.push_back(Entry); 1888 Entry.Node = Nest; Args.push_back(Entry); 1889 1890 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) 1891 TargetLowering::CallLoweringInfo CLI(DAG); 1892 CLI.setDebugLoc(dl).setChain(Chain) 1893 .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 1894 DAG.getExternalSymbol("__trampoline_setup", PtrVT), &Args, 0); 1895 1896 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 1897 return CallResult.second; 1898} 1899 1900SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG, 1901 const PPCSubtarget &Subtarget) const { 1902 MachineFunction &MF = DAG.getMachineFunction(); 1903 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 1904 1905 SDLoc dl(Op); 1906 1907 if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) { 1908 // vastart just stores the address of the VarArgsFrameIndex slot into the 1909 // memory location argument. 1910 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 1911 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 1912 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1913 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 1914 MachinePointerInfo(SV), 1915 false, false, 0); 1916 } 1917 1918 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. 1919 // We suppose the given va_list is already allocated. 1920 // 1921 // typedef struct { 1922 // char gpr; /* index into the array of 8 GPRs 1923 // * stored in the register save area 1924 // * gpr=0 corresponds to r3, 1925 // * gpr=1 to r4, etc. 1926 // */ 1927 // char fpr; /* index into the array of 8 FPRs 1928 // * stored in the register save area 1929 // * fpr=0 corresponds to f1, 1930 // * fpr=1 to f2, etc. 1931 // */ 1932 // char *overflow_arg_area; 1933 // /* location on stack that holds 1934 // * the next overflow argument 1935 // */ 1936 // char *reg_save_area; 1937 // /* where r3:r10 and f1:f8 (if saved) 1938 // * are stored 1939 // */ 1940 // } va_list[1]; 1941 1942 1943 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), MVT::i32); 1944 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), MVT::i32); 1945 1946 1947 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 1948 1949 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), 1950 PtrVT); 1951 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 1952 PtrVT); 1953 1954 uint64_t FrameOffset = PtrVT.getSizeInBits()/8; 1955 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, PtrVT); 1956 1957 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; 1958 SDValue ConstStackOffset = DAG.getConstant(StackOffset, PtrVT); 1959 1960 uint64_t FPROffset = 1; 1961 SDValue ConstFPROffset = DAG.getConstant(FPROffset, PtrVT); 1962 1963 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1964 1965 // Store first byte : number of int regs 1966 SDValue firstStore = DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, 1967 Op.getOperand(1), 1968 MachinePointerInfo(SV), 1969 MVT::i8, false, false, 0); 1970 uint64_t nextOffset = FPROffset; 1971 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), 1972 ConstFPROffset); 1973 1974 // Store second byte : number of float regs 1975 SDValue secondStore = 1976 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, 1977 MachinePointerInfo(SV, nextOffset), MVT::i8, 1978 false, false, 0); 1979 nextOffset += StackOffset; 1980 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); 1981 1982 // Store second word : arguments given on stack 1983 SDValue thirdStore = 1984 DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, 1985 MachinePointerInfo(SV, nextOffset), 1986 false, false, 0); 1987 nextOffset += FrameOffset; 1988 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); 1989 1990 // Store third word : arguments given in registers 1991 return DAG.getStore(thirdStore, dl, FR, nextPtr, 1992 MachinePointerInfo(SV, nextOffset), 1993 false, false, 0); 1994 1995} 1996 1997#include "PPCGenCallingConv.inc" 1998 1999// Function whose sole purpose is to kill compiler warnings 2000// stemming from unused functions included from PPCGenCallingConv.inc. 2001CCAssignFn *PPCTargetLowering::useFastISelCCs(unsigned Flag) const { 2002 return Flag ? CC_PPC64_ELF_FIS : RetCC_PPC64_ELF_FIS; 2003} 2004 2005bool llvm::CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT, 2006 CCValAssign::LocInfo &LocInfo, 2007 ISD::ArgFlagsTy &ArgFlags, 2008 CCState &State) { 2009 return true; 2010} 2011 2012bool llvm::CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT, 2013 MVT &LocVT, 2014 CCValAssign::LocInfo &LocInfo, 2015 ISD::ArgFlagsTy &ArgFlags, 2016 CCState &State) { 2017 static const MCPhysReg ArgRegs[] = { 2018 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 2019 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 2020 }; 2021 const unsigned NumArgRegs = array_lengthof(ArgRegs); 2022 2023 unsigned RegNum = State.getFirstUnallocated(ArgRegs, NumArgRegs); 2024 2025 // Skip one register if the first unallocated register has an even register 2026 // number and there are still argument registers available which have not been 2027 // allocated yet. RegNum is actually an index into ArgRegs, which means we 2028 // need to skip a register if RegNum is odd. 2029 if (RegNum != NumArgRegs && RegNum % 2 == 1) { 2030 State.AllocateReg(ArgRegs[RegNum]); 2031 } 2032 2033 // Always return false here, as this function only makes sure that the first 2034 // unallocated register has an odd register number and does not actually 2035 // allocate a register for the current argument. 2036 return false; 2037} 2038 2039bool llvm::CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT, 2040 MVT &LocVT, 2041 CCValAssign::LocInfo &LocInfo, 2042 ISD::ArgFlagsTy &ArgFlags, 2043 CCState &State) { 2044 static const MCPhysReg ArgRegs[] = { 2045 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 2046 PPC::F8 2047 }; 2048 2049 const unsigned NumArgRegs = array_lengthof(ArgRegs); 2050 2051 unsigned RegNum = State.getFirstUnallocated(ArgRegs, NumArgRegs); 2052 2053 // If there is only one Floating-point register left we need to put both f64 2054 // values of a split ppc_fp128 value on the stack. 2055 if (RegNum != NumArgRegs && ArgRegs[RegNum] == PPC::F8) { 2056 State.AllocateReg(ArgRegs[RegNum]); 2057 } 2058 2059 // Always return false here, as this function only makes sure that the two f64 2060 // values a ppc_fp128 value is split into are both passed in registers or both 2061 // passed on the stack and does not actually allocate a register for the 2062 // current argument. 2063 return false; 2064} 2065 2066/// GetFPR - Get the set of FP registers that should be allocated for arguments, 2067/// on Darwin. 2068static const MCPhysReg *GetFPR() { 2069 static const MCPhysReg FPR[] = { 2070 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 2071 PPC::F8, PPC::F9, PPC::F10, PPC::F11, PPC::F12, PPC::F13 2072 }; 2073 2074 return FPR; 2075} 2076 2077/// CalculateStackSlotSize - Calculates the size reserved for this argument on 2078/// the stack. 2079static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, 2080 unsigned PtrByteSize) { 2081 unsigned ArgSize = ArgVT.getStoreSize(); 2082 if (Flags.isByVal()) 2083 ArgSize = Flags.getByValSize(); 2084 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2085 2086 return ArgSize; 2087} 2088 2089SDValue 2090PPCTargetLowering::LowerFormalArguments(SDValue Chain, 2091 CallingConv::ID CallConv, bool isVarArg, 2092 const SmallVectorImpl<ISD::InputArg> 2093 &Ins, 2094 SDLoc dl, SelectionDAG &DAG, 2095 SmallVectorImpl<SDValue> &InVals) 2096 const { 2097 if (PPCSubTarget.isSVR4ABI()) { 2098 if (PPCSubTarget.isPPC64()) 2099 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, 2100 dl, DAG, InVals); 2101 else 2102 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, 2103 dl, DAG, InVals); 2104 } else { 2105 return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, 2106 dl, DAG, InVals); 2107 } 2108} 2109 2110SDValue 2111PPCTargetLowering::LowerFormalArguments_32SVR4( 2112 SDValue Chain, 2113 CallingConv::ID CallConv, bool isVarArg, 2114 const SmallVectorImpl<ISD::InputArg> 2115 &Ins, 2116 SDLoc dl, SelectionDAG &DAG, 2117 SmallVectorImpl<SDValue> &InVals) const { 2118 2119 // 32-bit SVR4 ABI Stack Frame Layout: 2120 // +-----------------------------------+ 2121 // +--> | Back chain | 2122 // | +-----------------------------------+ 2123 // | | Floating-point register save area | 2124 // | +-----------------------------------+ 2125 // | | General register save area | 2126 // | +-----------------------------------+ 2127 // | | CR save word | 2128 // | +-----------------------------------+ 2129 // | | VRSAVE save word | 2130 // | +-----------------------------------+ 2131 // | | Alignment padding | 2132 // | +-----------------------------------+ 2133 // | | Vector register save area | 2134 // | +-----------------------------------+ 2135 // | | Local variable space | 2136 // | +-----------------------------------+ 2137 // | | Parameter list area | 2138 // | +-----------------------------------+ 2139 // | | LR save word | 2140 // | +-----------------------------------+ 2141 // SP--> +--- | Back chain | 2142 // +-----------------------------------+ 2143 // 2144 // Specifications: 2145 // System V Application Binary Interface PowerPC Processor Supplement 2146 // AltiVec Technology Programming Interface Manual 2147 2148 MachineFunction &MF = DAG.getMachineFunction(); 2149 MachineFrameInfo *MFI = MF.getFrameInfo(); 2150 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2151 2152 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 2153 // Potential tail calls could cause overwriting of argument stack slots. 2154 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 2155 (CallConv == CallingConv::Fast)); 2156 unsigned PtrByteSize = 4; 2157 2158 // Assign locations to all of the incoming arguments. 2159 SmallVector<CCValAssign, 16> ArgLocs; 2160 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 2161 getTargetMachine(), ArgLocs, *DAG.getContext()); 2162 2163 // Reserve space for the linkage area on the stack. 2164 CCInfo.AllocateStack(PPCFrameLowering::getLinkageSize(false, false), PtrByteSize); 2165 2166 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); 2167 2168 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2169 CCValAssign &VA = ArgLocs[i]; 2170 2171 // Arguments stored in registers. 2172 if (VA.isRegLoc()) { 2173 const TargetRegisterClass *RC; 2174 EVT ValVT = VA.getValVT(); 2175 2176 switch (ValVT.getSimpleVT().SimpleTy) { 2177 default: 2178 llvm_unreachable("ValVT not supported by formal arguments Lowering"); 2179 case MVT::i1: 2180 case MVT::i32: 2181 RC = &PPC::GPRCRegClass; 2182 break; 2183 case MVT::f32: 2184 RC = &PPC::F4RCRegClass; 2185 break; 2186 case MVT::f64: 2187 if (PPCSubTarget.hasVSX()) 2188 RC = &PPC::VSFRCRegClass; 2189 else 2190 RC = &PPC::F8RCRegClass; 2191 break; 2192 case MVT::v16i8: 2193 case MVT::v8i16: 2194 case MVT::v4i32: 2195 case MVT::v4f32: 2196 RC = &PPC::VRRCRegClass; 2197 break; 2198 case MVT::v2f64: 2199 case MVT::v2i64: 2200 RC = &PPC::VSHRCRegClass; 2201 break; 2202 } 2203 2204 // Transform the arguments stored in physical registers into virtual ones. 2205 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 2206 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, 2207 ValVT == MVT::i1 ? MVT::i32 : ValVT); 2208 2209 if (ValVT == MVT::i1) 2210 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); 2211 2212 InVals.push_back(ArgValue); 2213 } else { 2214 // Argument stored in memory. 2215 assert(VA.isMemLoc()); 2216 2217 unsigned ArgSize = VA.getLocVT().getStoreSize(); 2218 int FI = MFI->CreateFixedObject(ArgSize, VA.getLocMemOffset(), 2219 isImmutable); 2220 2221 // Create load nodes to retrieve arguments from the stack. 2222 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2223 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN, 2224 MachinePointerInfo(), 2225 false, false, false, 0)); 2226 } 2227 } 2228 2229 // Assign locations to all of the incoming aggregate by value arguments. 2230 // Aggregates passed by value are stored in the local variable space of the 2231 // caller's stack frame, right above the parameter list area. 2232 SmallVector<CCValAssign, 16> ByValArgLocs; 2233 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 2234 getTargetMachine(), ByValArgLocs, *DAG.getContext()); 2235 2236 // Reserve stack space for the allocations in CCInfo. 2237 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 2238 2239 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); 2240 2241 // Area that is at least reserved in the caller of this function. 2242 unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); 2243 2244 // Set the size that is at least reserved in caller of this function. Tail 2245 // call optimized function's reserved stack space needs to be aligned so that 2246 // taking the difference between two stack areas will result in an aligned 2247 // stack. 2248 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 2249 2250 MinReservedArea = 2251 std::max(MinReservedArea, 2252 PPCFrameLowering::getMinCallFrameSize(false, false)); 2253 2254 unsigned TargetAlign = DAG.getMachineFunction().getTarget().getFrameLowering()-> 2255 getStackAlignment(); 2256 unsigned AlignMask = TargetAlign-1; 2257 MinReservedArea = (MinReservedArea + AlignMask) & ~AlignMask; 2258 2259 FI->setMinReservedArea(MinReservedArea); 2260 2261 SmallVector<SDValue, 8> MemOps; 2262 2263 // If the function takes variable number of arguments, make a frame index for 2264 // the start of the first vararg value... for expansion of llvm.va_start. 2265 if (isVarArg) { 2266 static const MCPhysReg GPArgRegs[] = { 2267 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 2268 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 2269 }; 2270 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); 2271 2272 static const MCPhysReg FPArgRegs[] = { 2273 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 2274 PPC::F8 2275 }; 2276 const unsigned NumFPArgRegs = array_lengthof(FPArgRegs); 2277 2278 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs, 2279 NumGPArgRegs)); 2280 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs, 2281 NumFPArgRegs)); 2282 2283 // Make room for NumGPArgRegs and NumFPArgRegs. 2284 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + 2285 NumFPArgRegs * EVT(MVT::f64).getSizeInBits()/8; 2286 2287 FuncInfo->setVarArgsStackOffset( 2288 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8, 2289 CCInfo.getNextStackOffset(), true)); 2290 2291 FuncInfo->setVarArgsFrameIndex(MFI->CreateStackObject(Depth, 8, false)); 2292 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 2293 2294 // The fixed integer arguments of a variadic function are stored to the 2295 // VarArgsFrameIndex on the stack so that they may be loaded by deferencing 2296 // the result of va_next. 2297 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { 2298 // Get an existing live-in vreg, or add a new one. 2299 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); 2300 if (!VReg) 2301 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); 2302 2303 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2304 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2305 MachinePointerInfo(), false, false, 0); 2306 MemOps.push_back(Store); 2307 // Increment the address by four for the next argument to store 2308 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, PtrVT); 2309 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 2310 } 2311 2312 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 2313 // is set. 2314 // The double arguments are stored to the VarArgsFrameIndex 2315 // on the stack. 2316 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { 2317 // Get an existing live-in vreg, or add a new one. 2318 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); 2319 if (!VReg) 2320 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); 2321 2322 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); 2323 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2324 MachinePointerInfo(), false, false, 0); 2325 MemOps.push_back(Store); 2326 // Increment the address by eight for the next argument to store 2327 SDValue PtrOff = DAG.getConstant(EVT(MVT::f64).getSizeInBits()/8, 2328 PtrVT); 2329 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 2330 } 2331 } 2332 2333 if (!MemOps.empty()) 2334 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 2335 2336 return Chain; 2337} 2338 2339// PPC64 passes i8, i16, and i32 values in i64 registers. Promote 2340// value to MVT::i64 and then truncate to the correct register size. 2341SDValue 2342PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, 2343 SelectionDAG &DAG, SDValue ArgVal, 2344 SDLoc dl) const { 2345 if (Flags.isSExt()) 2346 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, 2347 DAG.getValueType(ObjectVT)); 2348 else if (Flags.isZExt()) 2349 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, 2350 DAG.getValueType(ObjectVT)); 2351 2352 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); 2353} 2354 2355// Set the size that is at least reserved in caller of this function. Tail 2356// call optimized functions' reserved stack space needs to be aligned so that 2357// taking the difference between two stack areas will result in an aligned 2358// stack. 2359void 2360PPCTargetLowering::setMinReservedArea(MachineFunction &MF, SelectionDAG &DAG, 2361 unsigned nAltivecParamsAtEnd, 2362 unsigned MinReservedArea, 2363 bool isPPC64) const { 2364 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 2365 // Add the Altivec parameters at the end, if needed. 2366 if (nAltivecParamsAtEnd) { 2367 MinReservedArea = ((MinReservedArea+15)/16)*16; 2368 MinReservedArea += 16*nAltivecParamsAtEnd; 2369 } 2370 MinReservedArea = 2371 std::max(MinReservedArea, 2372 PPCFrameLowering::getMinCallFrameSize(isPPC64, true)); 2373 unsigned TargetAlign 2374 = DAG.getMachineFunction().getTarget().getFrameLowering()-> 2375 getStackAlignment(); 2376 unsigned AlignMask = TargetAlign-1; 2377 MinReservedArea = (MinReservedArea + AlignMask) & ~AlignMask; 2378 FI->setMinReservedArea(MinReservedArea); 2379} 2380 2381SDValue 2382PPCTargetLowering::LowerFormalArguments_64SVR4( 2383 SDValue Chain, 2384 CallingConv::ID CallConv, bool isVarArg, 2385 const SmallVectorImpl<ISD::InputArg> 2386 &Ins, 2387 SDLoc dl, SelectionDAG &DAG, 2388 SmallVectorImpl<SDValue> &InVals) const { 2389 // TODO: add description of PPC stack frame format, or at least some docs. 2390 // 2391 MachineFunction &MF = DAG.getMachineFunction(); 2392 MachineFrameInfo *MFI = MF.getFrameInfo(); 2393 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2394 2395 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 2396 // Potential tail calls could cause overwriting of argument stack slots. 2397 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 2398 (CallConv == CallingConv::Fast)); 2399 unsigned PtrByteSize = 8; 2400 2401 unsigned ArgOffset = PPCFrameLowering::getLinkageSize(true, true); 2402 // Area that is at least reserved in caller of this function. 2403 unsigned MinReservedArea = ArgOffset; 2404 2405 static const MCPhysReg GPR[] = { 2406 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 2407 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 2408 }; 2409 2410 static const MCPhysReg *FPR = GetFPR(); 2411 2412 static const MCPhysReg VR[] = { 2413 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 2414 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 2415 }; 2416 static const MCPhysReg VSRH[] = { 2417 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8, 2418 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13 2419 }; 2420 2421 const unsigned Num_GPR_Regs = array_lengthof(GPR); 2422 const unsigned Num_FPR_Regs = 13; 2423 const unsigned Num_VR_Regs = array_lengthof(VR); 2424 2425 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 2426 2427 // Add DAG nodes to load the arguments or copy them out of registers. On 2428 // entry to a function on PPC, the arguments start after the linkage area, 2429 // although the first ones are often in registers. 2430 2431 SmallVector<SDValue, 8> MemOps; 2432 unsigned nAltivecParamsAtEnd = 0; 2433 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin(); 2434 unsigned CurArgIdx = 0; 2435 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 2436 SDValue ArgVal; 2437 bool needsLoad = false; 2438 EVT ObjectVT = Ins[ArgNo].VT; 2439 unsigned ObjSize = ObjectVT.getStoreSize(); 2440 unsigned ArgSize = ObjSize; 2441 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 2442 std::advance(FuncArg, Ins[ArgNo].OrigArgIndex - CurArgIdx); 2443 CurArgIdx = Ins[ArgNo].OrigArgIndex; 2444 2445 unsigned CurArgOffset = ArgOffset; 2446 2447 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. 2448 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || 2449 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8 || 2450 ObjectVT==MVT::v2f64 || ObjectVT==MVT::v2i64) { 2451 if (isVarArg) { 2452 MinReservedArea = ((MinReservedArea+15)/16)*16; 2453 MinReservedArea += CalculateStackSlotSize(ObjectVT, 2454 Flags, 2455 PtrByteSize); 2456 } else 2457 nAltivecParamsAtEnd++; 2458 } else 2459 // Calculate min reserved area. 2460 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, 2461 Flags, 2462 PtrByteSize); 2463 2464 // FIXME the codegen can be much improved in some cases. 2465 // We do not have to keep everything in memory. 2466 if (Flags.isByVal()) { 2467 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 2468 ObjSize = Flags.getByValSize(); 2469 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2470 // Empty aggregate parameters do not take up registers. Examples: 2471 // struct { } a; 2472 // union { } b; 2473 // int c[0]; 2474 // etc. However, we have to provide a place-holder in InVals, so 2475 // pretend we have an 8-byte item at the current address for that 2476 // purpose. 2477 if (!ObjSize) { 2478 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 2479 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2480 InVals.push_back(FIN); 2481 continue; 2482 } 2483 2484 unsigned BVAlign = Flags.getByValAlign(); 2485 if (BVAlign > 8) { 2486 ArgOffset = ((ArgOffset+BVAlign-1)/BVAlign)*BVAlign; 2487 CurArgOffset = ArgOffset; 2488 } 2489 2490 // All aggregates smaller than 8 bytes must be passed right-justified. 2491 if (ObjSize < PtrByteSize) 2492 CurArgOffset = CurArgOffset + (PtrByteSize - ObjSize); 2493 // The value of the object is its address. 2494 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, true); 2495 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2496 InVals.push_back(FIN); 2497 2498 if (ObjSize < 8) { 2499 if (GPR_idx != Num_GPR_Regs) { 2500 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2501 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2502 SDValue Store; 2503 2504 if (ObjSize==1 || ObjSize==2 || ObjSize==4) { 2505 EVT ObjType = (ObjSize == 1 ? MVT::i8 : 2506 (ObjSize == 2 ? MVT::i16 : MVT::i32)); 2507 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, 2508 MachinePointerInfo(FuncArg), 2509 ObjType, false, false, 0); 2510 } else { 2511 // For sizes that don't fit a truncating store (3, 5, 6, 7), 2512 // store the whole register as-is to the parameter save area 2513 // slot. The address of the parameter was already calculated 2514 // above (InVals.push_back(FIN)) to be the right-justified 2515 // offset within the slot. For this store, we need a new 2516 // frame index that points at the beginning of the slot. 2517 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 2518 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2519 Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2520 MachinePointerInfo(FuncArg), 2521 false, false, 0); 2522 } 2523 2524 MemOps.push_back(Store); 2525 ++GPR_idx; 2526 } 2527 // Whether we copied from a register or not, advance the offset 2528 // into the parameter save area by a full doubleword. 2529 ArgOffset += PtrByteSize; 2530 continue; 2531 } 2532 2533 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 2534 // Store whatever pieces of the object are in registers 2535 // to memory. ArgOffset will be the address of the beginning 2536 // of the object. 2537 if (GPR_idx != Num_GPR_Regs) { 2538 unsigned VReg; 2539 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2540 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 2541 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2542 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2543 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2544 MachinePointerInfo(FuncArg, j), 2545 false, false, 0); 2546 MemOps.push_back(Store); 2547 ++GPR_idx; 2548 ArgOffset += PtrByteSize; 2549 } else { 2550 ArgOffset += ArgSize - j; 2551 break; 2552 } 2553 } 2554 continue; 2555 } 2556 2557 switch (ObjectVT.getSimpleVT().SimpleTy) { 2558 default: llvm_unreachable("Unhandled argument type!"); 2559 case MVT::i1: 2560 case MVT::i32: 2561 case MVT::i64: 2562 if (GPR_idx != Num_GPR_Regs) { 2563 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2564 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 2565 2566 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 2567 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 2568 // value to MVT::i64 and then truncate to the correct register size. 2569 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 2570 2571 ++GPR_idx; 2572 } else { 2573 needsLoad = true; 2574 ArgSize = PtrByteSize; 2575 } 2576 ArgOffset += 8; 2577 break; 2578 2579 case MVT::f32: 2580 case MVT::f64: 2581 // Every 8 bytes of argument space consumes one of the GPRs available for 2582 // argument passing. 2583 if (GPR_idx != Num_GPR_Regs) { 2584 ++GPR_idx; 2585 } 2586 if (FPR_idx != Num_FPR_Regs) { 2587 unsigned VReg; 2588 2589 if (ObjectVT == MVT::f32) 2590 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); 2591 else 2592 VReg = MF.addLiveIn(FPR[FPR_idx], PPCSubTarget.hasVSX() ? 2593 &PPC::VSFRCRegClass : 2594 &PPC::F8RCRegClass); 2595 2596 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 2597 ++FPR_idx; 2598 } else { 2599 needsLoad = true; 2600 ArgSize = PtrByteSize; 2601 } 2602 2603 ArgOffset += 8; 2604 break; 2605 case MVT::v4f32: 2606 case MVT::v4i32: 2607 case MVT::v8i16: 2608 case MVT::v16i8: 2609 case MVT::v2f64: 2610 case MVT::v2i64: 2611 // Note that vector arguments in registers don't reserve stack space, 2612 // except in varargs functions. 2613 if (VR_idx != Num_VR_Regs) { 2614 unsigned VReg = (ObjectVT == MVT::v2f64 || ObjectVT == MVT::v2i64) ? 2615 MF.addLiveIn(VSRH[VR_idx], &PPC::VSHRCRegClass) : 2616 MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 2617 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 2618 if (isVarArg) { 2619 while ((ArgOffset % 16) != 0) { 2620 ArgOffset += PtrByteSize; 2621 if (GPR_idx != Num_GPR_Regs) 2622 GPR_idx++; 2623 } 2624 ArgOffset += 16; 2625 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? 2626 } 2627 ++VR_idx; 2628 } else { 2629 // Vectors are aligned. 2630 ArgOffset = ((ArgOffset+15)/16)*16; 2631 CurArgOffset = ArgOffset; 2632 ArgOffset += 16; 2633 needsLoad = true; 2634 } 2635 break; 2636 } 2637 2638 // We need to load the argument to a virtual register if we determined 2639 // above that we ran out of physical registers of the appropriate type. 2640 if (needsLoad) { 2641 int FI = MFI->CreateFixedObject(ObjSize, 2642 CurArgOffset + (ArgSize - ObjSize), 2643 isImmutable); 2644 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2645 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(), 2646 false, false, false, 0); 2647 } 2648 2649 InVals.push_back(ArgVal); 2650 } 2651 2652 // Set the size that is at least reserved in caller of this function. Tail 2653 // call optimized functions' reserved stack space needs to be aligned so that 2654 // taking the difference between two stack areas will result in an aligned 2655 // stack. 2656 setMinReservedArea(MF, DAG, nAltivecParamsAtEnd, MinReservedArea, true); 2657 2658 // If the function takes variable number of arguments, make a frame index for 2659 // the start of the first vararg value... for expansion of llvm.va_start. 2660 if (isVarArg) { 2661 int Depth = ArgOffset; 2662 2663 FuncInfo->setVarArgsFrameIndex( 2664 MFI->CreateFixedObject(PtrByteSize, Depth, true)); 2665 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 2666 2667 // If this function is vararg, store any remaining integer argument regs 2668 // to their spots on the stack so that they may be loaded by deferencing the 2669 // result of va_next. 2670 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { 2671 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2672 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2673 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2674 MachinePointerInfo(), false, false, 0); 2675 MemOps.push_back(Store); 2676 // Increment the address by four for the next argument to store 2677 SDValue PtrOff = DAG.getConstant(PtrByteSize, PtrVT); 2678 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 2679 } 2680 } 2681 2682 if (!MemOps.empty()) 2683 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 2684 2685 return Chain; 2686} 2687 2688SDValue 2689PPCTargetLowering::LowerFormalArguments_Darwin( 2690 SDValue Chain, 2691 CallingConv::ID CallConv, bool isVarArg, 2692 const SmallVectorImpl<ISD::InputArg> 2693 &Ins, 2694 SDLoc dl, SelectionDAG &DAG, 2695 SmallVectorImpl<SDValue> &InVals) const { 2696 // TODO: add description of PPC stack frame format, or at least some docs. 2697 // 2698 MachineFunction &MF = DAG.getMachineFunction(); 2699 MachineFrameInfo *MFI = MF.getFrameInfo(); 2700 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2701 2702 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 2703 bool isPPC64 = PtrVT == MVT::i64; 2704 // Potential tail calls could cause overwriting of argument stack slots. 2705 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 2706 (CallConv == CallingConv::Fast)); 2707 unsigned PtrByteSize = isPPC64 ? 8 : 4; 2708 2709 unsigned ArgOffset = PPCFrameLowering::getLinkageSize(isPPC64, true); 2710 // Area that is at least reserved in caller of this function. 2711 unsigned MinReservedArea = ArgOffset; 2712 2713 static const MCPhysReg GPR_32[] = { // 32-bit registers. 2714 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 2715 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 2716 }; 2717 static const MCPhysReg GPR_64[] = { // 64-bit registers. 2718 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 2719 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 2720 }; 2721 2722 static const MCPhysReg *FPR = GetFPR(); 2723 2724 static const MCPhysReg VR[] = { 2725 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 2726 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 2727 }; 2728 2729 const unsigned Num_GPR_Regs = array_lengthof(GPR_32); 2730 const unsigned Num_FPR_Regs = 13; 2731 const unsigned Num_VR_Regs = array_lengthof( VR); 2732 2733 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 2734 2735 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 2736 2737 // In 32-bit non-varargs functions, the stack space for vectors is after the 2738 // stack space for non-vectors. We do not use this space unless we have 2739 // too many vectors to fit in registers, something that only occurs in 2740 // constructed examples:), but we have to walk the arglist to figure 2741 // that out...for the pathological case, compute VecArgOffset as the 2742 // start of the vector parameter area. Computing VecArgOffset is the 2743 // entire point of the following loop. 2744 unsigned VecArgOffset = ArgOffset; 2745 if (!isVarArg && !isPPC64) { 2746 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; 2747 ++ArgNo) { 2748 EVT ObjectVT = Ins[ArgNo].VT; 2749 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 2750 2751 if (Flags.isByVal()) { 2752 // ObjSize is the true size, ArgSize rounded up to multiple of regs. 2753 unsigned ObjSize = Flags.getByValSize(); 2754 unsigned ArgSize = 2755 ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2756 VecArgOffset += ArgSize; 2757 continue; 2758 } 2759 2760 switch(ObjectVT.getSimpleVT().SimpleTy) { 2761 default: llvm_unreachable("Unhandled argument type!"); 2762 case MVT::i1: 2763 case MVT::i32: 2764 case MVT::f32: 2765 VecArgOffset += 4; 2766 break; 2767 case MVT::i64: // PPC64 2768 case MVT::f64: 2769 // FIXME: We are guaranteed to be !isPPC64 at this point. 2770 // Does MVT::i64 apply? 2771 VecArgOffset += 8; 2772 break; 2773 case MVT::v4f32: 2774 case MVT::v4i32: 2775 case MVT::v8i16: 2776 case MVT::v16i8: 2777 // Nothing to do, we're only looking at Nonvector args here. 2778 break; 2779 } 2780 } 2781 } 2782 // We've found where the vector parameter area in memory is. Skip the 2783 // first 12 parameters; these don't use that memory. 2784 VecArgOffset = ((VecArgOffset+15)/16)*16; 2785 VecArgOffset += 12*16; 2786 2787 // Add DAG nodes to load the arguments or copy them out of registers. On 2788 // entry to a function on PPC, the arguments start after the linkage area, 2789 // although the first ones are often in registers. 2790 2791 SmallVector<SDValue, 8> MemOps; 2792 unsigned nAltivecParamsAtEnd = 0; 2793 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin(); 2794 unsigned CurArgIdx = 0; 2795 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 2796 SDValue ArgVal; 2797 bool needsLoad = false; 2798 EVT ObjectVT = Ins[ArgNo].VT; 2799 unsigned ObjSize = ObjectVT.getSizeInBits()/8; 2800 unsigned ArgSize = ObjSize; 2801 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 2802 std::advance(FuncArg, Ins[ArgNo].OrigArgIndex - CurArgIdx); 2803 CurArgIdx = Ins[ArgNo].OrigArgIndex; 2804 2805 unsigned CurArgOffset = ArgOffset; 2806 2807 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. 2808 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || 2809 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { 2810 if (isVarArg || isPPC64) { 2811 MinReservedArea = ((MinReservedArea+15)/16)*16; 2812 MinReservedArea += CalculateStackSlotSize(ObjectVT, 2813 Flags, 2814 PtrByteSize); 2815 } else nAltivecParamsAtEnd++; 2816 } else 2817 // Calculate min reserved area. 2818 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, 2819 Flags, 2820 PtrByteSize); 2821 2822 // FIXME the codegen can be much improved in some cases. 2823 // We do not have to keep everything in memory. 2824 if (Flags.isByVal()) { 2825 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 2826 ObjSize = Flags.getByValSize(); 2827 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 2828 // Objects of size 1 and 2 are right justified, everything else is 2829 // left justified. This means the memory address is adjusted forwards. 2830 if (ObjSize==1 || ObjSize==2) { 2831 CurArgOffset = CurArgOffset + (4 - ObjSize); 2832 } 2833 // The value of the object is its address. 2834 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, true); 2835 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2836 InVals.push_back(FIN); 2837 if (ObjSize==1 || ObjSize==2) { 2838 if (GPR_idx != Num_GPR_Regs) { 2839 unsigned VReg; 2840 if (isPPC64) 2841 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2842 else 2843 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 2844 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2845 EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; 2846 SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, 2847 MachinePointerInfo(FuncArg), 2848 ObjType, false, false, 0); 2849 MemOps.push_back(Store); 2850 ++GPR_idx; 2851 } 2852 2853 ArgOffset += PtrByteSize; 2854 2855 continue; 2856 } 2857 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 2858 // Store whatever pieces of the object are in registers 2859 // to memory. ArgOffset will be the address of the beginning 2860 // of the object. 2861 if (GPR_idx != Num_GPR_Regs) { 2862 unsigned VReg; 2863 if (isPPC64) 2864 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2865 else 2866 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 2867 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true); 2868 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2869 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 2870 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 2871 MachinePointerInfo(FuncArg, j), 2872 false, false, 0); 2873 MemOps.push_back(Store); 2874 ++GPR_idx; 2875 ArgOffset += PtrByteSize; 2876 } else { 2877 ArgOffset += ArgSize - (ArgOffset-CurArgOffset); 2878 break; 2879 } 2880 } 2881 continue; 2882 } 2883 2884 switch (ObjectVT.getSimpleVT().SimpleTy) { 2885 default: llvm_unreachable("Unhandled argument type!"); 2886 case MVT::i1: 2887 case MVT::i32: 2888 if (!isPPC64) { 2889 if (GPR_idx != Num_GPR_Regs) { 2890 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 2891 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 2892 2893 if (ObjectVT == MVT::i1) 2894 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); 2895 2896 ++GPR_idx; 2897 } else { 2898 needsLoad = true; 2899 ArgSize = PtrByteSize; 2900 } 2901 // All int arguments reserve stack space in the Darwin ABI. 2902 ArgOffset += PtrByteSize; 2903 break; 2904 } 2905 // FALLTHROUGH 2906 case MVT::i64: // PPC64 2907 if (GPR_idx != Num_GPR_Regs) { 2908 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 2909 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 2910 2911 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 2912 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 2913 // value to MVT::i64 and then truncate to the correct register size. 2914 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 2915 2916 ++GPR_idx; 2917 } else { 2918 needsLoad = true; 2919 ArgSize = PtrByteSize; 2920 } 2921 // All int arguments reserve stack space in the Darwin ABI. 2922 ArgOffset += 8; 2923 break; 2924 2925 case MVT::f32: 2926 case MVT::f64: 2927 // Every 4 bytes of argument space consumes one of the GPRs available for 2928 // argument passing. 2929 if (GPR_idx != Num_GPR_Regs) { 2930 ++GPR_idx; 2931 if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) 2932 ++GPR_idx; 2933 } 2934 if (FPR_idx != Num_FPR_Regs) { 2935 unsigned VReg; 2936 2937 if (ObjectVT == MVT::f32) 2938 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); 2939 else 2940 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); 2941 2942 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 2943 ++FPR_idx; 2944 } else { 2945 needsLoad = true; 2946 } 2947 2948 // All FP arguments reserve stack space in the Darwin ABI. 2949 ArgOffset += isPPC64 ? 8 : ObjSize; 2950 break; 2951 case MVT::v4f32: 2952 case MVT::v4i32: 2953 case MVT::v8i16: 2954 case MVT::v16i8: 2955 // Note that vector arguments in registers don't reserve stack space, 2956 // except in varargs functions. 2957 if (VR_idx != Num_VR_Regs) { 2958 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 2959 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 2960 if (isVarArg) { 2961 while ((ArgOffset % 16) != 0) { 2962 ArgOffset += PtrByteSize; 2963 if (GPR_idx != Num_GPR_Regs) 2964 GPR_idx++; 2965 } 2966 ArgOffset += 16; 2967 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? 2968 } 2969 ++VR_idx; 2970 } else { 2971 if (!isVarArg && !isPPC64) { 2972 // Vectors go after all the nonvectors. 2973 CurArgOffset = VecArgOffset; 2974 VecArgOffset += 16; 2975 } else { 2976 // Vectors are aligned. 2977 ArgOffset = ((ArgOffset+15)/16)*16; 2978 CurArgOffset = ArgOffset; 2979 ArgOffset += 16; 2980 } 2981 needsLoad = true; 2982 } 2983 break; 2984 } 2985 2986 // We need to load the argument to a virtual register if we determined above 2987 // that we ran out of physical registers of the appropriate type. 2988 if (needsLoad) { 2989 int FI = MFI->CreateFixedObject(ObjSize, 2990 CurArgOffset + (ArgSize - ObjSize), 2991 isImmutable); 2992 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 2993 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(), 2994 false, false, false, 0); 2995 } 2996 2997 InVals.push_back(ArgVal); 2998 } 2999 3000 // Set the size that is at least reserved in caller of this function. Tail 3001 // call optimized functions' reserved stack space needs to be aligned so that 3002 // taking the difference between two stack areas will result in an aligned 3003 // stack. 3004 setMinReservedArea(MF, DAG, nAltivecParamsAtEnd, MinReservedArea, isPPC64); 3005 3006 // If the function takes variable number of arguments, make a frame index for 3007 // the start of the first vararg value... for expansion of llvm.va_start. 3008 if (isVarArg) { 3009 int Depth = ArgOffset; 3010 3011 FuncInfo->setVarArgsFrameIndex( 3012 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8, 3013 Depth, true)); 3014 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3015 3016 // If this function is vararg, store any remaining integer argument regs 3017 // to their spots on the stack so that they may be loaded by deferencing the 3018 // result of va_next. 3019 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { 3020 unsigned VReg; 3021 3022 if (isPPC64) 3023 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3024 else 3025 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 3026 3027 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3028 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3029 MachinePointerInfo(), false, false, 0); 3030 MemOps.push_back(Store); 3031 // Increment the address by four for the next argument to store 3032 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, PtrVT); 3033 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3034 } 3035 } 3036 3037 if (!MemOps.empty()) 3038 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3039 3040 return Chain; 3041} 3042 3043/// CalculateParameterAndLinkageAreaSize - Get the size of the parameter plus 3044/// linkage area for the Darwin ABI, or the 64-bit SVR4 ABI. 3045static unsigned 3046CalculateParameterAndLinkageAreaSize(SelectionDAG &DAG, 3047 bool isPPC64, 3048 bool isVarArg, 3049 unsigned CC, 3050 const SmallVectorImpl<ISD::OutputArg> 3051 &Outs, 3052 const SmallVectorImpl<SDValue> &OutVals, 3053 unsigned &nAltivecParamsAtEnd) { 3054 // Count how many bytes are to be pushed on the stack, including the linkage 3055 // area, and parameter passing area. We start with 24/48 bytes, which is 3056 // prereserved space for [SP][CR][LR][3 x unused]. 3057 unsigned NumBytes = PPCFrameLowering::getLinkageSize(isPPC64, true); 3058 unsigned NumOps = Outs.size(); 3059 unsigned PtrByteSize = isPPC64 ? 8 : 4; 3060 3061 // Add up all the space actually used. 3062 // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually 3063 // they all go in registers, but we must reserve stack space for them for 3064 // possible use by the caller. In varargs or 64-bit calls, parameters are 3065 // assigned stack space in order, with padding so Altivec parameters are 3066 // 16-byte aligned. 3067 nAltivecParamsAtEnd = 0; 3068 for (unsigned i = 0; i != NumOps; ++i) { 3069 ISD::ArgFlagsTy Flags = Outs[i].Flags; 3070 EVT ArgVT = Outs[i].VT; 3071 // Varargs Altivec parameters are padded to a 16 byte boundary. 3072 if (ArgVT==MVT::v4f32 || ArgVT==MVT::v4i32 || 3073 ArgVT==MVT::v8i16 || ArgVT==MVT::v16i8 || 3074 ArgVT==MVT::v2f64 || ArgVT==MVT::v2i64) { 3075 if (!isVarArg && !isPPC64) { 3076 // Non-varargs Altivec parameters go after all the non-Altivec 3077 // parameters; handle those later so we know how much padding we need. 3078 nAltivecParamsAtEnd++; 3079 continue; 3080 } 3081 // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. 3082 NumBytes = ((NumBytes+15)/16)*16; 3083 } 3084 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 3085 } 3086 3087 // Allow for Altivec parameters at the end, if needed. 3088 if (nAltivecParamsAtEnd) { 3089 NumBytes = ((NumBytes+15)/16)*16; 3090 NumBytes += 16*nAltivecParamsAtEnd; 3091 } 3092 3093 // The prolog code of the callee may store up to 8 GPR argument registers to 3094 // the stack, allowing va_start to index over them in memory if its varargs. 3095 // Because we cannot tell if this is needed on the caller side, we have to 3096 // conservatively assume that it is needed. As such, make sure we have at 3097 // least enough stack space for the caller to store the 8 GPRs. 3098 NumBytes = std::max(NumBytes, 3099 PPCFrameLowering::getMinCallFrameSize(isPPC64, true)); 3100 3101 // Tail call needs the stack to be aligned. 3102 if (CC == CallingConv::Fast && DAG.getTarget().Options.GuaranteedTailCallOpt){ 3103 unsigned TargetAlign = DAG.getMachineFunction().getTarget(). 3104 getFrameLowering()->getStackAlignment(); 3105 unsigned AlignMask = TargetAlign-1; 3106 NumBytes = (NumBytes + AlignMask) & ~AlignMask; 3107 } 3108 3109 return NumBytes; 3110} 3111 3112/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be 3113/// adjusted to accommodate the arguments for the tailcall. 3114static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, 3115 unsigned ParamSize) { 3116 3117 if (!isTailCall) return 0; 3118 3119 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>(); 3120 unsigned CallerMinReservedArea = FI->getMinReservedArea(); 3121 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; 3122 // Remember only if the new adjustement is bigger. 3123 if (SPDiff < FI->getTailCallSPDelta()) 3124 FI->setTailCallSPDelta(SPDiff); 3125 3126 return SPDiff; 3127} 3128 3129/// IsEligibleForTailCallOptimization - Check whether the call is eligible 3130/// for tail call optimization. Targets which want to do tail call 3131/// optimization should implement this function. 3132bool 3133PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 3134 CallingConv::ID CalleeCC, 3135 bool isVarArg, 3136 const SmallVectorImpl<ISD::InputArg> &Ins, 3137 SelectionDAG& DAG) const { 3138 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 3139 return false; 3140 3141 // Variable argument functions are not supported. 3142 if (isVarArg) 3143 return false; 3144 3145 MachineFunction &MF = DAG.getMachineFunction(); 3146 CallingConv::ID CallerCC = MF.getFunction()->getCallingConv(); 3147 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 3148 // Functions containing by val parameters are not supported. 3149 for (unsigned i = 0; i != Ins.size(); i++) { 3150 ISD::ArgFlagsTy Flags = Ins[i].Flags; 3151 if (Flags.isByVal()) return false; 3152 } 3153 3154 // Non-PIC/GOT tail calls are supported. 3155 if (getTargetMachine().getRelocationModel() != Reloc::PIC_) 3156 return true; 3157 3158 // At the moment we can only do local tail calls (in same module, hidden 3159 // or protected) if we are generating PIC. 3160 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 3161 return G->getGlobal()->hasHiddenVisibility() 3162 || G->getGlobal()->hasProtectedVisibility(); 3163 } 3164 3165 return false; 3166} 3167 3168/// isCallCompatibleAddress - Return the immediate to use if the specified 3169/// 32-bit value is representable in the immediate field of a BxA instruction. 3170static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { 3171 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 3172 if (!C) return nullptr; 3173 3174 int Addr = C->getZExtValue(); 3175 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. 3176 SignExtend32<26>(Addr) != Addr) 3177 return nullptr; // Top 6 bits have to be sext of immediate. 3178 3179 return DAG.getConstant((int)C->getZExtValue() >> 2, 3180 DAG.getTargetLoweringInfo().getPointerTy()).getNode(); 3181} 3182 3183namespace { 3184 3185struct TailCallArgumentInfo { 3186 SDValue Arg; 3187 SDValue FrameIdxOp; 3188 int FrameIdx; 3189 3190 TailCallArgumentInfo() : FrameIdx(0) {} 3191}; 3192 3193} 3194 3195/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. 3196static void 3197StoreTailCallArgumentsToStackSlot(SelectionDAG &DAG, 3198 SDValue Chain, 3199 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs, 3200 SmallVectorImpl<SDValue> &MemOpChains, 3201 SDLoc dl) { 3202 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { 3203 SDValue Arg = TailCallArgs[i].Arg; 3204 SDValue FIN = TailCallArgs[i].FrameIdxOp; 3205 int FI = TailCallArgs[i].FrameIdx; 3206 // Store relative to framepointer. 3207 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, FIN, 3208 MachinePointerInfo::getFixedStack(FI), 3209 false, false, 0)); 3210 } 3211} 3212 3213/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to 3214/// the appropriate stack slot for the tail call optimized function call. 3215static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, 3216 MachineFunction &MF, 3217 SDValue Chain, 3218 SDValue OldRetAddr, 3219 SDValue OldFP, 3220 int SPDiff, 3221 bool isPPC64, 3222 bool isDarwinABI, 3223 SDLoc dl) { 3224 if (SPDiff) { 3225 // Calculate the new stack slot for the return address. 3226 int SlotSize = isPPC64 ? 8 : 4; 3227 int NewRetAddrLoc = SPDiff + PPCFrameLowering::getReturnSaveOffset(isPPC64, 3228 isDarwinABI); 3229 int NewRetAddr = MF.getFrameInfo()->CreateFixedObject(SlotSize, 3230 NewRetAddrLoc, true); 3231 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 3232 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); 3233 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, 3234 MachinePointerInfo::getFixedStack(NewRetAddr), 3235 false, false, 0); 3236 3237 // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack 3238 // slot as the FP is never overwritten. 3239 if (isDarwinABI) { 3240 int NewFPLoc = 3241 SPDiff + PPCFrameLowering::getFramePointerSaveOffset(isPPC64, isDarwinABI); 3242 int NewFPIdx = MF.getFrameInfo()->CreateFixedObject(SlotSize, NewFPLoc, 3243 true); 3244 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT); 3245 Chain = DAG.getStore(Chain, dl, OldFP, NewFramePtrIdx, 3246 MachinePointerInfo::getFixedStack(NewFPIdx), 3247 false, false, 0); 3248 } 3249 } 3250 return Chain; 3251} 3252 3253/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate 3254/// the position of the argument. 3255static void 3256CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, 3257 SDValue Arg, int SPDiff, unsigned ArgOffset, 3258 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) { 3259 int Offset = ArgOffset + SPDiff; 3260 uint32_t OpSize = (Arg.getValueType().getSizeInBits()+7)/8; 3261 int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); 3262 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 3263 SDValue FIN = DAG.getFrameIndex(FI, VT); 3264 TailCallArgumentInfo Info; 3265 Info.Arg = Arg; 3266 Info.FrameIdxOp = FIN; 3267 Info.FrameIdx = FI; 3268 TailCallArguments.push_back(Info); 3269} 3270 3271/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address 3272/// stack slot. Returns the chain as result and the loaded frame pointers in 3273/// LROpOut/FPOpout. Used when tail calling. 3274SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(SelectionDAG & DAG, 3275 int SPDiff, 3276 SDValue Chain, 3277 SDValue &LROpOut, 3278 SDValue &FPOpOut, 3279 bool isDarwinABI, 3280 SDLoc dl) const { 3281 if (SPDiff) { 3282 // Load the LR and FP stack slot for later adjusting. 3283 EVT VT = PPCSubTarget.isPPC64() ? MVT::i64 : MVT::i32; 3284 LROpOut = getReturnAddrFrameIndex(DAG); 3285 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo(), 3286 false, false, false, 0); 3287 Chain = SDValue(LROpOut.getNode(), 1); 3288 3289 // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack 3290 // slot as the FP is never overwritten. 3291 if (isDarwinABI) { 3292 FPOpOut = getFramePointerFrameIndex(DAG); 3293 FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo(), 3294 false, false, false, 0); 3295 Chain = SDValue(FPOpOut.getNode(), 1); 3296 } 3297 } 3298 return Chain; 3299} 3300 3301/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 3302/// by "Src" to address "Dst" of size "Size". Alignment information is 3303/// specified by the specific parameter attribute. The copy will be passed as 3304/// a byval function parameter. 3305/// Sometimes what we are copying is the end of a larger object, the part that 3306/// does not fit in registers. 3307static SDValue 3308CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 3309 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, 3310 SDLoc dl) { 3311 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 3312 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 3313 false, false, MachinePointerInfo(), 3314 MachinePointerInfo()); 3315} 3316 3317/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of 3318/// tail calls. 3319static void 3320LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, 3321 SDValue Arg, SDValue PtrOff, int SPDiff, 3322 unsigned ArgOffset, bool isPPC64, bool isTailCall, 3323 bool isVector, SmallVectorImpl<SDValue> &MemOpChains, 3324 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, 3325 SDLoc dl) { 3326 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 3327 if (!isTailCall) { 3328 if (isVector) { 3329 SDValue StackPtr; 3330 if (isPPC64) 3331 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 3332 else 3333 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 3334 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 3335 DAG.getConstant(ArgOffset, PtrVT)); 3336 } 3337 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, 3338 MachinePointerInfo(), false, false, 0)); 3339 // Calculate and remember argument location. 3340 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, 3341 TailCallArguments); 3342} 3343 3344static 3345void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, 3346 SDLoc dl, bool isPPC64, int SPDiff, unsigned NumBytes, 3347 SDValue LROp, SDValue FPOp, bool isDarwinABI, 3348 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) { 3349 MachineFunction &MF = DAG.getMachineFunction(); 3350 3351 // Emit a sequence of copyto/copyfrom virtual registers for arguments that 3352 // might overwrite each other in case of tail call optimization. 3353 SmallVector<SDValue, 8> MemOpChains2; 3354 // Do not flag preceding copytoreg stuff together with the following stuff. 3355 InFlag = SDValue(); 3356 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, 3357 MemOpChains2, dl); 3358 if (!MemOpChains2.empty()) 3359 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); 3360 3361 // Store the return address to the appropriate stack slot. 3362 Chain = EmitTailCallStoreFPAndRetAddr(DAG, MF, Chain, LROp, FPOp, SPDiff, 3363 isPPC64, isDarwinABI, dl); 3364 3365 // Emit callseq_end just before tailcall node. 3366 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), 3367 DAG.getIntPtrConstant(0, true), InFlag, dl); 3368 InFlag = Chain.getValue(1); 3369} 3370 3371static 3372unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, 3373 SDValue &Chain, SDLoc dl, int SPDiff, bool isTailCall, 3374 SmallVectorImpl<std::pair<unsigned, SDValue> > &RegsToPass, 3375 SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys, 3376 const PPCSubtarget &PPCSubTarget) { 3377 3378 bool isPPC64 = PPCSubTarget.isPPC64(); 3379 bool isSVR4ABI = PPCSubTarget.isSVR4ABI(); 3380 3381 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 3382 NodeTys.push_back(MVT::Other); // Returns a chain 3383 NodeTys.push_back(MVT::Glue); // Returns a flag for retval copy to use. 3384 3385 unsigned CallOpc = PPCISD::CALL; 3386 3387 bool needIndirectCall = true; 3388 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) { 3389 // If this is an absolute destination address, use the munged value. 3390 Callee = SDValue(Dest, 0); 3391 needIndirectCall = false; 3392 } 3393 3394 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 3395 // XXX Work around for http://llvm.org/bugs/show_bug.cgi?id=5201 3396 // Use indirect calls for ALL functions calls in JIT mode, since the 3397 // far-call stubs may be outside relocation limits for a BL instruction. 3398 if (!DAG.getTarget().getSubtarget<PPCSubtarget>().isJITCodeModel()) { 3399 unsigned OpFlags = 0; 3400 if (DAG.getTarget().getRelocationModel() != Reloc::Static && 3401 (PPCSubTarget.getTargetTriple().isMacOSX() && 3402 PPCSubTarget.getTargetTriple().isMacOSXVersionLT(10, 5)) && 3403 (G->getGlobal()->isDeclaration() || 3404 G->getGlobal()->isWeakForLinker())) { 3405 // PC-relative references to external symbols should go through $stub, 3406 // unless we're building with the leopard linker or later, which 3407 // automatically synthesizes these stubs. 3408 OpFlags = PPCII::MO_DARWIN_STUB; 3409 } 3410 3411 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, 3412 // every direct call is) turn it into a TargetGlobalAddress / 3413 // TargetExternalSymbol node so that legalize doesn't hack it. 3414 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, 3415 Callee.getValueType(), 3416 0, OpFlags); 3417 needIndirectCall = false; 3418 } 3419 } 3420 3421 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 3422 unsigned char OpFlags = 0; 3423 3424 if (DAG.getTarget().getRelocationModel() != Reloc::Static && 3425 (PPCSubTarget.getTargetTriple().isMacOSX() && 3426 PPCSubTarget.getTargetTriple().isMacOSXVersionLT(10, 5))) { 3427 // PC-relative references to external symbols should go through $stub, 3428 // unless we're building with the leopard linker or later, which 3429 // automatically synthesizes these stubs. 3430 OpFlags = PPCII::MO_DARWIN_STUB; 3431 } 3432 3433 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), Callee.getValueType(), 3434 OpFlags); 3435 needIndirectCall = false; 3436 } 3437 3438 if (needIndirectCall) { 3439 // Otherwise, this is an indirect call. We have to use a MTCTR/BCTRL pair 3440 // to do the call, we can't use PPCISD::CALL. 3441 SDValue MTCTROps[] = {Chain, Callee, InFlag}; 3442 3443 if (isSVR4ABI && isPPC64) { 3444 // Function pointers in the 64-bit SVR4 ABI do not point to the function 3445 // entry point, but to the function descriptor (the function entry point 3446 // address is part of the function descriptor though). 3447 // The function descriptor is a three doubleword structure with the 3448 // following fields: function entry point, TOC base address and 3449 // environment pointer. 3450 // Thus for a call through a function pointer, the following actions need 3451 // to be performed: 3452 // 1. Save the TOC of the caller in the TOC save area of its stack 3453 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). 3454 // 2. Load the address of the function entry point from the function 3455 // descriptor. 3456 // 3. Load the TOC of the callee from the function descriptor into r2. 3457 // 4. Load the environment pointer from the function descriptor into 3458 // r11. 3459 // 5. Branch to the function entry point address. 3460 // 6. On return of the callee, the TOC of the caller needs to be 3461 // restored (this is done in FinishCall()). 3462 // 3463 // All those operations are flagged together to ensure that no other 3464 // operations can be scheduled in between. E.g. without flagging the 3465 // operations together, a TOC access in the caller could be scheduled 3466 // between the load of the callee TOC and the branch to the callee, which 3467 // results in the TOC access going through the TOC of the callee instead 3468 // of going through the TOC of the caller, which leads to incorrect code. 3469 3470 // Load the address of the function entry point from the function 3471 // descriptor. 3472 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other, MVT::Glue); 3473 SDValue LoadFuncPtr = DAG.getNode(PPCISD::LOAD, dl, VTs, 3474 makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2)); 3475 Chain = LoadFuncPtr.getValue(1); 3476 InFlag = LoadFuncPtr.getValue(2); 3477 3478 // Load environment pointer into r11. 3479 // Offset of the environment pointer within the function descriptor. 3480 SDValue PtrOff = DAG.getIntPtrConstant(16); 3481 3482 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff); 3483 SDValue LoadEnvPtr = DAG.getNode(PPCISD::LOAD, dl, VTs, Chain, AddPtr, 3484 InFlag); 3485 Chain = LoadEnvPtr.getValue(1); 3486 InFlag = LoadEnvPtr.getValue(2); 3487 3488 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr, 3489 InFlag); 3490 Chain = EnvVal.getValue(0); 3491 InFlag = EnvVal.getValue(1); 3492 3493 // Load TOC of the callee into r2. We are using a target-specific load 3494 // with r2 hard coded, because the result of a target-independent load 3495 // would never go directly into r2, since r2 is a reserved register (which 3496 // prevents the register allocator from allocating it), resulting in an 3497 // additional register being allocated and an unnecessary move instruction 3498 // being generated. 3499 VTs = DAG.getVTList(MVT::Other, MVT::Glue); 3500 SDValue LoadTOCPtr = DAG.getNode(PPCISD::LOAD_TOC, dl, VTs, Chain, 3501 Callee, InFlag); 3502 Chain = LoadTOCPtr.getValue(0); 3503 InFlag = LoadTOCPtr.getValue(1); 3504 3505 MTCTROps[0] = Chain; 3506 MTCTROps[1] = LoadFuncPtr; 3507 MTCTROps[2] = InFlag; 3508 } 3509 3510 Chain = DAG.getNode(PPCISD::MTCTR, dl, NodeTys, 3511 makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2)); 3512 InFlag = Chain.getValue(1); 3513 3514 NodeTys.clear(); 3515 NodeTys.push_back(MVT::Other); 3516 NodeTys.push_back(MVT::Glue); 3517 Ops.push_back(Chain); 3518 CallOpc = PPCISD::BCTRL; 3519 Callee.setNode(nullptr); 3520 // Add use of X11 (holding environment pointer) 3521 if (isSVR4ABI && isPPC64) 3522 Ops.push_back(DAG.getRegister(PPC::X11, PtrVT)); 3523 // Add CTR register as callee so a bctr can be emitted later. 3524 if (isTailCall) 3525 Ops.push_back(DAG.getRegister(isPPC64 ? PPC::CTR8 : PPC::CTR, PtrVT)); 3526 } 3527 3528 // If this is a direct call, pass the chain and the callee. 3529 if (Callee.getNode()) { 3530 Ops.push_back(Chain); 3531 Ops.push_back(Callee); 3532 } 3533 // If this is a tail call add stack pointer delta. 3534 if (isTailCall) 3535 Ops.push_back(DAG.getConstant(SPDiff, MVT::i32)); 3536 3537 // Add argument registers to the end of the list so that they are known live 3538 // into the call. 3539 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 3540 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 3541 RegsToPass[i].second.getValueType())); 3542 3543 return CallOpc; 3544} 3545 3546static 3547bool isLocalCall(const SDValue &Callee) 3548{ 3549 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 3550 return !G->getGlobal()->isDeclaration() && 3551 !G->getGlobal()->isWeakForLinker(); 3552 return false; 3553} 3554 3555SDValue 3556PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag, 3557 CallingConv::ID CallConv, bool isVarArg, 3558 const SmallVectorImpl<ISD::InputArg> &Ins, 3559 SDLoc dl, SelectionDAG &DAG, 3560 SmallVectorImpl<SDValue> &InVals) const { 3561 3562 SmallVector<CCValAssign, 16> RVLocs; 3563 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), 3564 getTargetMachine(), RVLocs, *DAG.getContext()); 3565 CCRetInfo.AnalyzeCallResult(Ins, RetCC_PPC); 3566 3567 // Copy all of the result registers out of their specified physreg. 3568 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 3569 CCValAssign &VA = RVLocs[i]; 3570 assert(VA.isRegLoc() && "Can only return in registers!"); 3571 3572 SDValue Val = DAG.getCopyFromReg(Chain, dl, 3573 VA.getLocReg(), VA.getLocVT(), InFlag); 3574 Chain = Val.getValue(1); 3575 InFlag = Val.getValue(2); 3576 3577 switch (VA.getLocInfo()) { 3578 default: llvm_unreachable("Unknown loc info!"); 3579 case CCValAssign::Full: break; 3580 case CCValAssign::AExt: 3581 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 3582 break; 3583 case CCValAssign::ZExt: 3584 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, 3585 DAG.getValueType(VA.getValVT())); 3586 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 3587 break; 3588 case CCValAssign::SExt: 3589 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, 3590 DAG.getValueType(VA.getValVT())); 3591 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 3592 break; 3593 } 3594 3595 InVals.push_back(Val); 3596 } 3597 3598 return Chain; 3599} 3600 3601SDValue 3602PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl, 3603 bool isTailCall, bool isVarArg, 3604 SelectionDAG &DAG, 3605 SmallVector<std::pair<unsigned, SDValue>, 8> 3606 &RegsToPass, 3607 SDValue InFlag, SDValue Chain, 3608 SDValue &Callee, 3609 int SPDiff, unsigned NumBytes, 3610 const SmallVectorImpl<ISD::InputArg> &Ins, 3611 SmallVectorImpl<SDValue> &InVals) const { 3612 std::vector<EVT> NodeTys; 3613 SmallVector<SDValue, 8> Ops; 3614 unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, dl, SPDiff, 3615 isTailCall, RegsToPass, Ops, NodeTys, 3616 PPCSubTarget); 3617 3618 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls 3619 if (isVarArg && PPCSubTarget.isSVR4ABI() && !PPCSubTarget.isPPC64()) 3620 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); 3621 3622 // When performing tail call optimization the callee pops its arguments off 3623 // the stack. Account for this here so these bytes can be pushed back on in 3624 // PPCFrameLowering::eliminateCallFramePseudoInstr. 3625 int BytesCalleePops = 3626 (CallConv == CallingConv::Fast && 3627 getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0; 3628 3629 // Add a register mask operand representing the call-preserved registers. 3630 const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo(); 3631 const uint32_t *Mask = TRI->getCallPreservedMask(CallConv); 3632 assert(Mask && "Missing call preserved mask for calling convention"); 3633 Ops.push_back(DAG.getRegisterMask(Mask)); 3634 3635 if (InFlag.getNode()) 3636 Ops.push_back(InFlag); 3637 3638 // Emit tail call. 3639 if (isTailCall) { 3640 assert(((Callee.getOpcode() == ISD::Register && 3641 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || 3642 Callee.getOpcode() == ISD::TargetExternalSymbol || 3643 Callee.getOpcode() == ISD::TargetGlobalAddress || 3644 isa<ConstantSDNode>(Callee)) && 3645 "Expecting an global address, external symbol, absolute value or register"); 3646 3647 return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops); 3648 } 3649 3650 // Add a NOP immediately after the branch instruction when using the 64-bit 3651 // SVR4 ABI. At link time, if caller and callee are in a different module and 3652 // thus have a different TOC, the call will be replaced with a call to a stub 3653 // function which saves the current TOC, loads the TOC of the callee and 3654 // branches to the callee. The NOP will be replaced with a load instruction 3655 // which restores the TOC of the caller from the TOC save slot of the current 3656 // stack frame. If caller and callee belong to the same module (and have the 3657 // same TOC), the NOP will remain unchanged. 3658 3659 bool needsTOCRestore = false; 3660 if (!isTailCall && PPCSubTarget.isSVR4ABI()&& PPCSubTarget.isPPC64()) { 3661 if (CallOpc == PPCISD::BCTRL) { 3662 // This is a call through a function pointer. 3663 // Restore the caller TOC from the save area into R2. 3664 // See PrepareCall() for more information about calls through function 3665 // pointers in the 64-bit SVR4 ABI. 3666 // We are using a target-specific load with r2 hard coded, because the 3667 // result of a target-independent load would never go directly into r2, 3668 // since r2 is a reserved register (which prevents the register allocator 3669 // from allocating it), resulting in an additional register being 3670 // allocated and an unnecessary move instruction being generated. 3671 needsTOCRestore = true; 3672 } else if ((CallOpc == PPCISD::CALL) && 3673 (!isLocalCall(Callee) || 3674 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) { 3675 // Otherwise insert NOP for non-local calls. 3676 CallOpc = PPCISD::CALL_NOP; 3677 } 3678 } 3679 3680 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); 3681 InFlag = Chain.getValue(1); 3682 3683 if (needsTOCRestore) { 3684 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 3685 Chain = DAG.getNode(PPCISD::TOC_RESTORE, dl, VTs, Chain, InFlag); 3686 InFlag = Chain.getValue(1); 3687 } 3688 3689 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), 3690 DAG.getIntPtrConstant(BytesCalleePops, true), 3691 InFlag, dl); 3692 if (!Ins.empty()) 3693 InFlag = Chain.getValue(1); 3694 3695 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 3696 Ins, dl, DAG, InVals); 3697} 3698 3699SDValue 3700PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 3701 SmallVectorImpl<SDValue> &InVals) const { 3702 SelectionDAG &DAG = CLI.DAG; 3703 SDLoc &dl = CLI.DL; 3704 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 3705 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 3706 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 3707 SDValue Chain = CLI.Chain; 3708 SDValue Callee = CLI.Callee; 3709 bool &isTailCall = CLI.IsTailCall; 3710 CallingConv::ID CallConv = CLI.CallConv; 3711 bool isVarArg = CLI.IsVarArg; 3712 3713 if (isTailCall) 3714 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, 3715 Ins, DAG); 3716 3717 if (!isTailCall && CLI.CS && CLI.CS->isMustTailCall()) 3718 report_fatal_error("failed to perform tail call elimination on a call " 3719 "site marked musttail"); 3720 3721 if (PPCSubTarget.isSVR4ABI()) { 3722 if (PPCSubTarget.isPPC64()) 3723 return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg, 3724 isTailCall, Outs, OutVals, Ins, 3725 dl, DAG, InVals); 3726 else 3727 return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg, 3728 isTailCall, Outs, OutVals, Ins, 3729 dl, DAG, InVals); 3730 } 3731 3732 return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg, 3733 isTailCall, Outs, OutVals, Ins, 3734 dl, DAG, InVals); 3735} 3736 3737SDValue 3738PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee, 3739 CallingConv::ID CallConv, bool isVarArg, 3740 bool isTailCall, 3741 const SmallVectorImpl<ISD::OutputArg> &Outs, 3742 const SmallVectorImpl<SDValue> &OutVals, 3743 const SmallVectorImpl<ISD::InputArg> &Ins, 3744 SDLoc dl, SelectionDAG &DAG, 3745 SmallVectorImpl<SDValue> &InVals) const { 3746 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description 3747 // of the 32-bit SVR4 ABI stack frame layout. 3748 3749 assert((CallConv == CallingConv::C || 3750 CallConv == CallingConv::Fast) && "Unknown calling convention!"); 3751 3752 unsigned PtrByteSize = 4; 3753 3754 MachineFunction &MF = DAG.getMachineFunction(); 3755 3756 // Mark this function as potentially containing a function that contains a 3757 // tail call. As a consequence the frame pointer will be used for dynamicalloc 3758 // and restoring the callers stack pointer in this functions epilog. This is 3759 // done because by tail calling the called function might overwrite the value 3760 // in this function's (MF) stack pointer stack slot 0(SP). 3761 if (getTargetMachine().Options.GuaranteedTailCallOpt && 3762 CallConv == CallingConv::Fast) 3763 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 3764 3765 // Count how many bytes are to be pushed on the stack, including the linkage 3766 // area, parameter list area and the part of the local variable space which 3767 // contains copies of aggregates which are passed by value. 3768 3769 // Assign locations to all of the outgoing arguments. 3770 SmallVector<CCValAssign, 16> ArgLocs; 3771 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 3772 getTargetMachine(), ArgLocs, *DAG.getContext()); 3773 3774 // Reserve space for the linkage area on the stack. 3775 CCInfo.AllocateStack(PPCFrameLowering::getLinkageSize(false, false), PtrByteSize); 3776 3777 if (isVarArg) { 3778 // Handle fixed and variable vector arguments differently. 3779 // Fixed vector arguments go into registers as long as registers are 3780 // available. Variable vector arguments always go into memory. 3781 unsigned NumArgs = Outs.size(); 3782 3783 for (unsigned i = 0; i != NumArgs; ++i) { 3784 MVT ArgVT = Outs[i].VT; 3785 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 3786 bool Result; 3787 3788 if (Outs[i].IsFixed) { 3789 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, 3790 CCInfo); 3791 } else { 3792 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, 3793 ArgFlags, CCInfo); 3794 } 3795 3796 if (Result) { 3797#ifndef NDEBUG 3798 errs() << "Call operand #" << i << " has unhandled type " 3799 << EVT(ArgVT).getEVTString() << "\n"; 3800#endif 3801 llvm_unreachable(nullptr); 3802 } 3803 } 3804 } else { 3805 // All arguments are treated the same. 3806 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); 3807 } 3808 3809 // Assign locations to all of the outgoing aggregate by value arguments. 3810 SmallVector<CCValAssign, 16> ByValArgLocs; 3811 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 3812 getTargetMachine(), ByValArgLocs, *DAG.getContext()); 3813 3814 // Reserve stack space for the allocations in CCInfo. 3815 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 3816 3817 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); 3818 3819 // Size of the linkage area, parameter list area and the part of the local 3820 // space variable where copies of aggregates which are passed by value are 3821 // stored. 3822 unsigned NumBytes = CCByValInfo.getNextStackOffset(); 3823 3824 // Calculate by how many bytes the stack has to be adjusted in case of tail 3825 // call optimization. 3826 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 3827 3828 // Adjust the stack pointer for the new arguments... 3829 // These operations are automatically eliminated by the prolog/epilog pass 3830 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), 3831 dl); 3832 SDValue CallSeqStart = Chain; 3833 3834 // Load the return address and frame pointer so it can be moved somewhere else 3835 // later. 3836 SDValue LROp, FPOp; 3837 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, false, 3838 dl); 3839 3840 // Set up a copy of the stack pointer for use loading and storing any 3841 // arguments that may not fit in the registers available for argument 3842 // passing. 3843 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 3844 3845 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 3846 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 3847 SmallVector<SDValue, 8> MemOpChains; 3848 3849 bool seenFloatArg = false; 3850 // Walk the register/memloc assignments, inserting copies/loads. 3851 for (unsigned i = 0, j = 0, e = ArgLocs.size(); 3852 i != e; 3853 ++i) { 3854 CCValAssign &VA = ArgLocs[i]; 3855 SDValue Arg = OutVals[i]; 3856 ISD::ArgFlagsTy Flags = Outs[i].Flags; 3857 3858 if (Flags.isByVal()) { 3859 // Argument is an aggregate which is passed by value, thus we need to 3860 // create a copy of it in the local variable space of the current stack 3861 // frame (which is the stack frame of the caller) and pass the address of 3862 // this copy to the callee. 3863 assert((j < ByValArgLocs.size()) && "Index out of bounds!"); 3864 CCValAssign &ByValVA = ByValArgLocs[j++]; 3865 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); 3866 3867 // Memory reserved in the local variable space of the callers stack frame. 3868 unsigned LocMemOffset = ByValVA.getLocMemOffset(); 3869 3870 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 3871 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); 3872 3873 // Create a copy of the argument in the local area of the current 3874 // stack frame. 3875 SDValue MemcpyCall = 3876 CreateCopyOfByValArgument(Arg, PtrOff, 3877 CallSeqStart.getNode()->getOperand(0), 3878 Flags, DAG, dl); 3879 3880 // This must go outside the CALLSEQ_START..END. 3881 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, 3882 CallSeqStart.getNode()->getOperand(1), 3883 SDLoc(MemcpyCall)); 3884 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 3885 NewCallSeqStart.getNode()); 3886 Chain = CallSeqStart = NewCallSeqStart; 3887 3888 // Pass the address of the aggregate copy on the stack either in a 3889 // physical register or in the parameter list area of the current stack 3890 // frame to the callee. 3891 Arg = PtrOff; 3892 } 3893 3894 if (VA.isRegLoc()) { 3895 if (Arg.getValueType() == MVT::i1) 3896 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Arg); 3897 3898 seenFloatArg |= VA.getLocVT().isFloatingPoint(); 3899 // Put argument in a physical register. 3900 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 3901 } else { 3902 // Put argument in the parameter list area of the current stack frame. 3903 assert(VA.isMemLoc()); 3904 unsigned LocMemOffset = VA.getLocMemOffset(); 3905 3906 if (!isTailCall) { 3907 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 3908 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); 3909 3910 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, 3911 MachinePointerInfo(), 3912 false, false, 0)); 3913 } else { 3914 // Calculate and remember argument location. 3915 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, 3916 TailCallArguments); 3917 } 3918 } 3919 } 3920 3921 if (!MemOpChains.empty()) 3922 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 3923 3924 // Build a sequence of copy-to-reg nodes chained together with token chain 3925 // and flag operands which copy the outgoing args into the appropriate regs. 3926 SDValue InFlag; 3927 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 3928 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 3929 RegsToPass[i].second, InFlag); 3930 InFlag = Chain.getValue(1); 3931 } 3932 3933 // Set CR bit 6 to true if this is a vararg call with floating args passed in 3934 // registers. 3935 if (isVarArg) { 3936 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 3937 SDValue Ops[] = { Chain, InFlag }; 3938 3939 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, 3940 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); 3941 3942 InFlag = Chain.getValue(1); 3943 } 3944 3945 if (isTailCall) 3946 PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp, 3947 false, TailCallArguments); 3948 3949 return FinishCall(CallConv, dl, isTailCall, isVarArg, DAG, 3950 RegsToPass, InFlag, Chain, Callee, SPDiff, NumBytes, 3951 Ins, InVals); 3952} 3953 3954// Copy an argument into memory, being careful to do this outside the 3955// call sequence for the call to which the argument belongs. 3956SDValue 3957PPCTargetLowering::createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff, 3958 SDValue CallSeqStart, 3959 ISD::ArgFlagsTy Flags, 3960 SelectionDAG &DAG, 3961 SDLoc dl) const { 3962 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, 3963 CallSeqStart.getNode()->getOperand(0), 3964 Flags, DAG, dl); 3965 // The MEMCPY must go outside the CALLSEQ_START..END. 3966 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, 3967 CallSeqStart.getNode()->getOperand(1), 3968 SDLoc(MemcpyCall)); 3969 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 3970 NewCallSeqStart.getNode()); 3971 return NewCallSeqStart; 3972} 3973 3974SDValue 3975PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee, 3976 CallingConv::ID CallConv, bool isVarArg, 3977 bool isTailCall, 3978 const SmallVectorImpl<ISD::OutputArg> &Outs, 3979 const SmallVectorImpl<SDValue> &OutVals, 3980 const SmallVectorImpl<ISD::InputArg> &Ins, 3981 SDLoc dl, SelectionDAG &DAG, 3982 SmallVectorImpl<SDValue> &InVals) const { 3983 3984 unsigned NumOps = Outs.size(); 3985 3986 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 3987 unsigned PtrByteSize = 8; 3988 3989 MachineFunction &MF = DAG.getMachineFunction(); 3990 3991 // Mark this function as potentially containing a function that contains a 3992 // tail call. As a consequence the frame pointer will be used for dynamicalloc 3993 // and restoring the callers stack pointer in this functions epilog. This is 3994 // done because by tail calling the called function might overwrite the value 3995 // in this function's (MF) stack pointer stack slot 0(SP). 3996 if (getTargetMachine().Options.GuaranteedTailCallOpt && 3997 CallConv == CallingConv::Fast) 3998 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 3999 4000 unsigned nAltivecParamsAtEnd = 0; 4001 4002 // Count how many bytes are to be pushed on the stack, including the linkage 4003 // area, and parameter passing area. We start with at least 48 bytes, which 4004 // is reserved space for [SP][CR][LR][3 x unused]. 4005 // NOTE: For PPC64, nAltivecParamsAtEnd always remains zero as a result 4006 // of this call. 4007 unsigned NumBytes = 4008 CalculateParameterAndLinkageAreaSize(DAG, true, isVarArg, CallConv, 4009 Outs, OutVals, nAltivecParamsAtEnd); 4010 4011 // Calculate by how many bytes the stack has to be adjusted in case of tail 4012 // call optimization. 4013 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 4014 4015 // To protect arguments on the stack from being clobbered in a tail call, 4016 // force all the loads to happen before doing any other lowering. 4017 if (isTailCall) 4018 Chain = DAG.getStackArgumentTokenFactor(Chain); 4019 4020 // Adjust the stack pointer for the new arguments... 4021 // These operations are automatically eliminated by the prolog/epilog pass 4022 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), 4023 dl); 4024 SDValue CallSeqStart = Chain; 4025 4026 // Load the return address and frame pointer so it can be move somewhere else 4027 // later. 4028 SDValue LROp, FPOp; 4029 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true, 4030 dl); 4031 4032 // Set up a copy of the stack pointer for use loading and storing any 4033 // arguments that may not fit in the registers available for argument 4034 // passing. 4035 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4036 4037 // Figure out which arguments are going to go in registers, and which in 4038 // memory. Also, if this is a vararg function, floating point operations 4039 // must be stored to our stack, and loaded into integer regs as well, if 4040 // any integer regs are available for argument passing. 4041 unsigned ArgOffset = PPCFrameLowering::getLinkageSize(true, true); 4042 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4043 4044 static const MCPhysReg GPR[] = { 4045 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4046 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4047 }; 4048 static const MCPhysReg *FPR = GetFPR(); 4049 4050 static const MCPhysReg VR[] = { 4051 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4052 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4053 }; 4054 static const MCPhysReg VSRH[] = { 4055 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8, 4056 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13 4057 }; 4058 4059 const unsigned NumGPRs = array_lengthof(GPR); 4060 const unsigned NumFPRs = 13; 4061 const unsigned NumVRs = array_lengthof(VR); 4062 4063 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 4064 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 4065 4066 SmallVector<SDValue, 8> MemOpChains; 4067 for (unsigned i = 0; i != NumOps; ++i) { 4068 SDValue Arg = OutVals[i]; 4069 ISD::ArgFlagsTy Flags = Outs[i].Flags; 4070 4071 // PtrOff will be used to store the current argument to the stack if a 4072 // register cannot be found for it. 4073 SDValue PtrOff; 4074 4075 PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType()); 4076 4077 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 4078 4079 // Promote integers to 64-bit values. 4080 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { 4081 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 4082 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4083 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 4084 } 4085 4086 // FIXME memcpy is used way more than necessary. Correctness first. 4087 // Note: "by value" is code for passing a structure by value, not 4088 // basic types. 4089 if (Flags.isByVal()) { 4090 // Note: Size includes alignment padding, so 4091 // struct x { short a; char b; } 4092 // will have Size = 4. With #pragma pack(1), it will have Size = 3. 4093 // These are the proper values we need for right-justifying the 4094 // aggregate in a parameter register. 4095 unsigned Size = Flags.getByValSize(); 4096 4097 // An empty aggregate parameter takes up no storage and no 4098 // registers. 4099 if (Size == 0) 4100 continue; 4101 4102 unsigned BVAlign = Flags.getByValAlign(); 4103 if (BVAlign > 8) { 4104 if (BVAlign % PtrByteSize != 0) 4105 llvm_unreachable( 4106 "ByVal alignment is not a multiple of the pointer size"); 4107 4108 ArgOffset = ((ArgOffset+BVAlign-1)/BVAlign)*BVAlign; 4109 } 4110 4111 // All aggregates smaller than 8 bytes must be passed right-justified. 4112 if (Size==1 || Size==2 || Size==4) { 4113 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); 4114 if (GPR_idx != NumGPRs) { 4115 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 4116 MachinePointerInfo(), VT, 4117 false, false, 0); 4118 MemOpChains.push_back(Load.getValue(1)); 4119 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4120 4121 ArgOffset += PtrByteSize; 4122 continue; 4123 } 4124 } 4125 4126 if (GPR_idx == NumGPRs && Size < 8) { 4127 SDValue Const = DAG.getConstant(PtrByteSize - Size, 4128 PtrOff.getValueType()); 4129 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 4130 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 4131 CallSeqStart, 4132 Flags, DAG, dl); 4133 ArgOffset += PtrByteSize; 4134 continue; 4135 } 4136 // Copy entire object into memory. There are cases where gcc-generated 4137 // code assumes it is there, even if it could be put entirely into 4138 // registers. (This is not what the doc says.) 4139 4140 // FIXME: The above statement is likely due to a misunderstanding of the 4141 // documents. All arguments must be copied into the parameter area BY 4142 // THE CALLEE in the event that the callee takes the address of any 4143 // formal argument. That has not yet been implemented. However, it is 4144 // reasonable to use the stack area as a staging area for the register 4145 // load. 4146 4147 // Skip this for small aggregates, as we will use the same slot for a 4148 // right-justified copy, below. 4149 if (Size >= 8) 4150 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 4151 CallSeqStart, 4152 Flags, DAG, dl); 4153 4154 // When a register is available, pass a small aggregate right-justified. 4155 if (Size < 8 && GPR_idx != NumGPRs) { 4156 // The easiest way to get this right-justified in a register 4157 // is to copy the structure into the rightmost portion of a 4158 // local variable slot, then load the whole slot into the 4159 // register. 4160 // FIXME: The memcpy seems to produce pretty awful code for 4161 // small aggregates, particularly for packed ones. 4162 // FIXME: It would be preferable to use the slot in the 4163 // parameter save area instead of a new local variable. 4164 SDValue Const = DAG.getConstant(8 - Size, PtrOff.getValueType()); 4165 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 4166 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 4167 CallSeqStart, 4168 Flags, DAG, dl); 4169 4170 // Load the slot into the register. 4171 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff, 4172 MachinePointerInfo(), 4173 false, false, false, 0); 4174 MemOpChains.push_back(Load.getValue(1)); 4175 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4176 4177 // Done with this argument. 4178 ArgOffset += PtrByteSize; 4179 continue; 4180 } 4181 4182 // For aggregates larger than PtrByteSize, copy the pieces of the 4183 // object that fit into registers from the parameter save area. 4184 for (unsigned j=0; j<Size; j+=PtrByteSize) { 4185 SDValue Const = DAG.getConstant(j, PtrOff.getValueType()); 4186 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 4187 if (GPR_idx != NumGPRs) { 4188 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 4189 MachinePointerInfo(), 4190 false, false, false, 0); 4191 MemOpChains.push_back(Load.getValue(1)); 4192 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4193 ArgOffset += PtrByteSize; 4194 } else { 4195 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 4196 break; 4197 } 4198 } 4199 continue; 4200 } 4201 4202 switch (Arg.getSimpleValueType().SimpleTy) { 4203 default: llvm_unreachable("Unexpected ValueType for argument!"); 4204 case MVT::i1: 4205 case MVT::i32: 4206 case MVT::i64: 4207 if (GPR_idx != NumGPRs) { 4208 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 4209 } else { 4210 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4211 true, isTailCall, false, MemOpChains, 4212 TailCallArguments, dl); 4213 } 4214 ArgOffset += PtrByteSize; 4215 break; 4216 case MVT::f32: 4217 case MVT::f64: 4218 if (FPR_idx != NumFPRs) { 4219 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 4220 4221 if (isVarArg) { 4222 // A single float or an aggregate containing only a single float 4223 // must be passed right-justified in the stack doubleword, and 4224 // in the GPR, if one is available. 4225 SDValue StoreOff; 4226 if (Arg.getSimpleValueType().SimpleTy == MVT::f32) { 4227 SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType()); 4228 StoreOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 4229 } else 4230 StoreOff = PtrOff; 4231 4232 SDValue Store = DAG.getStore(Chain, dl, Arg, StoreOff, 4233 MachinePointerInfo(), false, false, 0); 4234 MemOpChains.push_back(Store); 4235 4236 // Float varargs are always shadowed in available integer registers 4237 if (GPR_idx != NumGPRs) { 4238 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff, 4239 MachinePointerInfo(), false, false, 4240 false, 0); 4241 MemOpChains.push_back(Load.getValue(1)); 4242 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4243 } 4244 } else if (GPR_idx != NumGPRs) 4245 // If we have any FPRs remaining, we may also have GPRs remaining. 4246 ++GPR_idx; 4247 } else { 4248 // Single-precision floating-point values are mapped to the 4249 // second (rightmost) word of the stack doubleword. 4250 if (Arg.getValueType() == MVT::f32) { 4251 SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType()); 4252 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 4253 } 4254 4255 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4256 true, isTailCall, false, MemOpChains, 4257 TailCallArguments, dl); 4258 } 4259 ArgOffset += 8; 4260 break; 4261 case MVT::v4f32: 4262 case MVT::v4i32: 4263 case MVT::v8i16: 4264 case MVT::v16i8: 4265 case MVT::v2f64: 4266 case MVT::v2i64: 4267 if (isVarArg) { 4268 // These go aligned on the stack, or in the corresponding R registers 4269 // when within range. The Darwin PPC ABI doc claims they also go in 4270 // V registers; in fact gcc does this only for arguments that are 4271 // prototyped, not for those that match the ... We do it for all 4272 // arguments, seems to work. 4273 while (ArgOffset % 16 !=0) { 4274 ArgOffset += PtrByteSize; 4275 if (GPR_idx != NumGPRs) 4276 GPR_idx++; 4277 } 4278 // We could elide this store in the case where the object fits 4279 // entirely in R registers. Maybe later. 4280 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 4281 DAG.getConstant(ArgOffset, PtrVT)); 4282 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 4283 MachinePointerInfo(), false, false, 0); 4284 MemOpChains.push_back(Store); 4285 if (VR_idx != NumVRs) { 4286 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, 4287 MachinePointerInfo(), 4288 false, false, false, 0); 4289 MemOpChains.push_back(Load.getValue(1)); 4290 4291 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 || 4292 Arg.getSimpleValueType() == MVT::v2i64) ? 4293 VSRH[VR_idx] : VR[VR_idx]; 4294 ++VR_idx; 4295 4296 RegsToPass.push_back(std::make_pair(VReg, Load)); 4297 } 4298 ArgOffset += 16; 4299 for (unsigned i=0; i<16; i+=PtrByteSize) { 4300 if (GPR_idx == NumGPRs) 4301 break; 4302 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 4303 DAG.getConstant(i, PtrVT)); 4304 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(), 4305 false, false, false, 0); 4306 MemOpChains.push_back(Load.getValue(1)); 4307 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4308 } 4309 break; 4310 } 4311 4312 // Non-varargs Altivec params generally go in registers, but have 4313 // stack space allocated at the end. 4314 if (VR_idx != NumVRs) { 4315 // Doesn't have GPR space allocated. 4316 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 || 4317 Arg.getSimpleValueType() == MVT::v2i64) ? 4318 VSRH[VR_idx] : VR[VR_idx]; 4319 ++VR_idx; 4320 4321 RegsToPass.push_back(std::make_pair(VReg, Arg)); 4322 } else { 4323 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4324 true, isTailCall, true, MemOpChains, 4325 TailCallArguments, dl); 4326 ArgOffset += 16; 4327 } 4328 break; 4329 } 4330 } 4331 4332 if (!MemOpChains.empty()) 4333 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 4334 4335 // Check if this is an indirect call (MTCTR/BCTRL). 4336 // See PrepareCall() for more information about calls through function 4337 // pointers in the 64-bit SVR4 ABI. 4338 if (!isTailCall && 4339 !dyn_cast<GlobalAddressSDNode>(Callee) && 4340 !dyn_cast<ExternalSymbolSDNode>(Callee) && 4341 !isBLACompatibleAddress(Callee, DAG)) { 4342 // Load r2 into a virtual register and store it to the TOC save area. 4343 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); 4344 // TOC save area offset. 4345 SDValue PtrOff = DAG.getIntPtrConstant(40); 4346 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 4347 Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo(), 4348 false, false, 0); 4349 // R12 must contain the address of an indirect callee. This does not 4350 // mean the MTCTR instruction must use R12; it's easier to model this 4351 // as an extra parameter, so do that. 4352 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); 4353 } 4354 4355 // Build a sequence of copy-to-reg nodes chained together with token chain 4356 // and flag operands which copy the outgoing args into the appropriate regs. 4357 SDValue InFlag; 4358 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 4359 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 4360 RegsToPass[i].second, InFlag); 4361 InFlag = Chain.getValue(1); 4362 } 4363 4364 if (isTailCall) 4365 PrepareTailCall(DAG, InFlag, Chain, dl, true, SPDiff, NumBytes, LROp, 4366 FPOp, true, TailCallArguments); 4367 4368 return FinishCall(CallConv, dl, isTailCall, isVarArg, DAG, 4369 RegsToPass, InFlag, Chain, Callee, SPDiff, NumBytes, 4370 Ins, InVals); 4371} 4372 4373SDValue 4374PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee, 4375 CallingConv::ID CallConv, bool isVarArg, 4376 bool isTailCall, 4377 const SmallVectorImpl<ISD::OutputArg> &Outs, 4378 const SmallVectorImpl<SDValue> &OutVals, 4379 const SmallVectorImpl<ISD::InputArg> &Ins, 4380 SDLoc dl, SelectionDAG &DAG, 4381 SmallVectorImpl<SDValue> &InVals) const { 4382 4383 unsigned NumOps = Outs.size(); 4384 4385 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 4386 bool isPPC64 = PtrVT == MVT::i64; 4387 unsigned PtrByteSize = isPPC64 ? 8 : 4; 4388 4389 MachineFunction &MF = DAG.getMachineFunction(); 4390 4391 // Mark this function as potentially containing a function that contains a 4392 // tail call. As a consequence the frame pointer will be used for dynamicalloc 4393 // and restoring the callers stack pointer in this functions epilog. This is 4394 // done because by tail calling the called function might overwrite the value 4395 // in this function's (MF) stack pointer stack slot 0(SP). 4396 if (getTargetMachine().Options.GuaranteedTailCallOpt && 4397 CallConv == CallingConv::Fast) 4398 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 4399 4400 unsigned nAltivecParamsAtEnd = 0; 4401 4402 // Count how many bytes are to be pushed on the stack, including the linkage 4403 // area, and parameter passing area. We start with 24/48 bytes, which is 4404 // prereserved space for [SP][CR][LR][3 x unused]. 4405 unsigned NumBytes = 4406 CalculateParameterAndLinkageAreaSize(DAG, isPPC64, isVarArg, CallConv, 4407 Outs, OutVals, 4408 nAltivecParamsAtEnd); 4409 4410 // Calculate by how many bytes the stack has to be adjusted in case of tail 4411 // call optimization. 4412 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 4413 4414 // To protect arguments on the stack from being clobbered in a tail call, 4415 // force all the loads to happen before doing any other lowering. 4416 if (isTailCall) 4417 Chain = DAG.getStackArgumentTokenFactor(Chain); 4418 4419 // Adjust the stack pointer for the new arguments... 4420 // These operations are automatically eliminated by the prolog/epilog pass 4421 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), 4422 dl); 4423 SDValue CallSeqStart = Chain; 4424 4425 // Load the return address and frame pointer so it can be move somewhere else 4426 // later. 4427 SDValue LROp, FPOp; 4428 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true, 4429 dl); 4430 4431 // Set up a copy of the stack pointer for use loading and storing any 4432 // arguments that may not fit in the registers available for argument 4433 // passing. 4434 SDValue StackPtr; 4435 if (isPPC64) 4436 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4437 else 4438 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 4439 4440 // Figure out which arguments are going to go in registers, and which in 4441 // memory. Also, if this is a vararg function, floating point operations 4442 // must be stored to our stack, and loaded into integer regs as well, if 4443 // any integer regs are available for argument passing. 4444 unsigned ArgOffset = PPCFrameLowering::getLinkageSize(isPPC64, true); 4445 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4446 4447 static const MCPhysReg GPR_32[] = { // 32-bit registers. 4448 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 4449 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 4450 }; 4451 static const MCPhysReg GPR_64[] = { // 64-bit registers. 4452 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4453 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4454 }; 4455 static const MCPhysReg *FPR = GetFPR(); 4456 4457 static const MCPhysReg VR[] = { 4458 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4459 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4460 }; 4461 const unsigned NumGPRs = array_lengthof(GPR_32); 4462 const unsigned NumFPRs = 13; 4463 const unsigned NumVRs = array_lengthof(VR); 4464 4465 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 4466 4467 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 4468 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 4469 4470 SmallVector<SDValue, 8> MemOpChains; 4471 for (unsigned i = 0; i != NumOps; ++i) { 4472 SDValue Arg = OutVals[i]; 4473 ISD::ArgFlagsTy Flags = Outs[i].Flags; 4474 4475 // PtrOff will be used to store the current argument to the stack if a 4476 // register cannot be found for it. 4477 SDValue PtrOff; 4478 4479 PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType()); 4480 4481 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 4482 4483 // On PPC64, promote integers to 64-bit values. 4484 if (isPPC64 && Arg.getValueType() == MVT::i32) { 4485 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 4486 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4487 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 4488 } 4489 4490 // FIXME memcpy is used way more than necessary. Correctness first. 4491 // Note: "by value" is code for passing a structure by value, not 4492 // basic types. 4493 if (Flags.isByVal()) { 4494 unsigned Size = Flags.getByValSize(); 4495 // Very small objects are passed right-justified. Everything else is 4496 // passed left-justified. 4497 if (Size==1 || Size==2) { 4498 EVT VT = (Size==1) ? MVT::i8 : MVT::i16; 4499 if (GPR_idx != NumGPRs) { 4500 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 4501 MachinePointerInfo(), VT, 4502 false, false, 0); 4503 MemOpChains.push_back(Load.getValue(1)); 4504 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4505 4506 ArgOffset += PtrByteSize; 4507 } else { 4508 SDValue Const = DAG.getConstant(PtrByteSize - Size, 4509 PtrOff.getValueType()); 4510 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 4511 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 4512 CallSeqStart, 4513 Flags, DAG, dl); 4514 ArgOffset += PtrByteSize; 4515 } 4516 continue; 4517 } 4518 // Copy entire object into memory. There are cases where gcc-generated 4519 // code assumes it is there, even if it could be put entirely into 4520 // registers. (This is not what the doc says.) 4521 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 4522 CallSeqStart, 4523 Flags, DAG, dl); 4524 4525 // For small aggregates (Darwin only) and aggregates >= PtrByteSize, 4526 // copy the pieces of the object that fit into registers from the 4527 // parameter save area. 4528 for (unsigned j=0; j<Size; j+=PtrByteSize) { 4529 SDValue Const = DAG.getConstant(j, PtrOff.getValueType()); 4530 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 4531 if (GPR_idx != NumGPRs) { 4532 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg, 4533 MachinePointerInfo(), 4534 false, false, false, 0); 4535 MemOpChains.push_back(Load.getValue(1)); 4536 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4537 ArgOffset += PtrByteSize; 4538 } else { 4539 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 4540 break; 4541 } 4542 } 4543 continue; 4544 } 4545 4546 switch (Arg.getSimpleValueType().SimpleTy) { 4547 default: llvm_unreachable("Unexpected ValueType for argument!"); 4548 case MVT::i1: 4549 case MVT::i32: 4550 case MVT::i64: 4551 if (GPR_idx != NumGPRs) { 4552 if (Arg.getValueType() == MVT::i1) 4553 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg); 4554 4555 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 4556 } else { 4557 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4558 isPPC64, isTailCall, false, MemOpChains, 4559 TailCallArguments, dl); 4560 } 4561 ArgOffset += PtrByteSize; 4562 break; 4563 case MVT::f32: 4564 case MVT::f64: 4565 if (FPR_idx != NumFPRs) { 4566 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 4567 4568 if (isVarArg) { 4569 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 4570 MachinePointerInfo(), false, false, 0); 4571 MemOpChains.push_back(Store); 4572 4573 // Float varargs are always shadowed in available integer registers 4574 if (GPR_idx != NumGPRs) { 4575 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff, 4576 MachinePointerInfo(), false, false, 4577 false, 0); 4578 MemOpChains.push_back(Load.getValue(1)); 4579 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4580 } 4581 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){ 4582 SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType()); 4583 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 4584 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff, 4585 MachinePointerInfo(), 4586 false, false, false, 0); 4587 MemOpChains.push_back(Load.getValue(1)); 4588 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4589 } 4590 } else { 4591 // If we have any FPRs remaining, we may also have GPRs remaining. 4592 // Args passed in FPRs consume either 1 (f32) or 2 (f64) available 4593 // GPRs. 4594 if (GPR_idx != NumGPRs) 4595 ++GPR_idx; 4596 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && 4597 !isPPC64) // PPC64 has 64-bit GPR's obviously :) 4598 ++GPR_idx; 4599 } 4600 } else 4601 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4602 isPPC64, isTailCall, false, MemOpChains, 4603 TailCallArguments, dl); 4604 if (isPPC64) 4605 ArgOffset += 8; 4606 else 4607 ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8; 4608 break; 4609 case MVT::v4f32: 4610 case MVT::v4i32: 4611 case MVT::v8i16: 4612 case MVT::v16i8: 4613 if (isVarArg) { 4614 // These go aligned on the stack, or in the corresponding R registers 4615 // when within range. The Darwin PPC ABI doc claims they also go in 4616 // V registers; in fact gcc does this only for arguments that are 4617 // prototyped, not for those that match the ... We do it for all 4618 // arguments, seems to work. 4619 while (ArgOffset % 16 !=0) { 4620 ArgOffset += PtrByteSize; 4621 if (GPR_idx != NumGPRs) 4622 GPR_idx++; 4623 } 4624 // We could elide this store in the case where the object fits 4625 // entirely in R registers. Maybe later. 4626 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 4627 DAG.getConstant(ArgOffset, PtrVT)); 4628 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, 4629 MachinePointerInfo(), false, false, 0); 4630 MemOpChains.push_back(Store); 4631 if (VR_idx != NumVRs) { 4632 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, 4633 MachinePointerInfo(), 4634 false, false, false, 0); 4635 MemOpChains.push_back(Load.getValue(1)); 4636 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 4637 } 4638 ArgOffset += 16; 4639 for (unsigned i=0; i<16; i+=PtrByteSize) { 4640 if (GPR_idx == NumGPRs) 4641 break; 4642 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 4643 DAG.getConstant(i, PtrVT)); 4644 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(), 4645 false, false, false, 0); 4646 MemOpChains.push_back(Load.getValue(1)); 4647 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 4648 } 4649 break; 4650 } 4651 4652 // Non-varargs Altivec params generally go in registers, but have 4653 // stack space allocated at the end. 4654 if (VR_idx != NumVRs) { 4655 // Doesn't have GPR space allocated. 4656 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 4657 } else if (nAltivecParamsAtEnd==0) { 4658 // We are emitting Altivec params in order. 4659 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4660 isPPC64, isTailCall, true, MemOpChains, 4661 TailCallArguments, dl); 4662 ArgOffset += 16; 4663 } 4664 break; 4665 } 4666 } 4667 // If all Altivec parameters fit in registers, as they usually do, 4668 // they get stack space following the non-Altivec parameters. We 4669 // don't track this here because nobody below needs it. 4670 // If there are more Altivec parameters than fit in registers emit 4671 // the stores here. 4672 if (!isVarArg && nAltivecParamsAtEnd > NumVRs) { 4673 unsigned j = 0; 4674 // Offset is aligned; skip 1st 12 params which go in V registers. 4675 ArgOffset = ((ArgOffset+15)/16)*16; 4676 ArgOffset += 12*16; 4677 for (unsigned i = 0; i != NumOps; ++i) { 4678 SDValue Arg = OutVals[i]; 4679 EVT ArgType = Outs[i].VT; 4680 if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || 4681 ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { 4682 if (++j > NumVRs) { 4683 SDValue PtrOff; 4684 // We are emitting Altivec params in order. 4685 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 4686 isPPC64, isTailCall, true, MemOpChains, 4687 TailCallArguments, dl); 4688 ArgOffset += 16; 4689 } 4690 } 4691 } 4692 } 4693 4694 if (!MemOpChains.empty()) 4695 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 4696 4697 // On Darwin, R12 must contain the address of an indirect callee. This does 4698 // not mean the MTCTR instruction must use R12; it's easier to model this as 4699 // an extra parameter, so do that. 4700 if (!isTailCall && 4701 !dyn_cast<GlobalAddressSDNode>(Callee) && 4702 !dyn_cast<ExternalSymbolSDNode>(Callee) && 4703 !isBLACompatibleAddress(Callee, DAG)) 4704 RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : 4705 PPC::R12), Callee)); 4706 4707 // Build a sequence of copy-to-reg nodes chained together with token chain 4708 // and flag operands which copy the outgoing args into the appropriate regs. 4709 SDValue InFlag; 4710 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 4711 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 4712 RegsToPass[i].second, InFlag); 4713 InFlag = Chain.getValue(1); 4714 } 4715 4716 if (isTailCall) 4717 PrepareTailCall(DAG, InFlag, Chain, dl, isPPC64, SPDiff, NumBytes, LROp, 4718 FPOp, true, TailCallArguments); 4719 4720 return FinishCall(CallConv, dl, isTailCall, isVarArg, DAG, 4721 RegsToPass, InFlag, Chain, Callee, SPDiff, NumBytes, 4722 Ins, InVals); 4723} 4724 4725bool 4726PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 4727 MachineFunction &MF, bool isVarArg, 4728 const SmallVectorImpl<ISD::OutputArg> &Outs, 4729 LLVMContext &Context) const { 4730 SmallVector<CCValAssign, 16> RVLocs; 4731 CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), 4732 RVLocs, Context); 4733 return CCInfo.CheckReturn(Outs, RetCC_PPC); 4734} 4735 4736SDValue 4737PPCTargetLowering::LowerReturn(SDValue Chain, 4738 CallingConv::ID CallConv, bool isVarArg, 4739 const SmallVectorImpl<ISD::OutputArg> &Outs, 4740 const SmallVectorImpl<SDValue> &OutVals, 4741 SDLoc dl, SelectionDAG &DAG) const { 4742 4743 SmallVector<CCValAssign, 16> RVLocs; 4744 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 4745 getTargetMachine(), RVLocs, *DAG.getContext()); 4746 CCInfo.AnalyzeReturn(Outs, RetCC_PPC); 4747 4748 SDValue Flag; 4749 SmallVector<SDValue, 4> RetOps(1, Chain); 4750 4751 // Copy the result values into the output registers. 4752 for (unsigned i = 0; i != RVLocs.size(); ++i) { 4753 CCValAssign &VA = RVLocs[i]; 4754 assert(VA.isRegLoc() && "Can only return in registers!"); 4755 4756 SDValue Arg = OutVals[i]; 4757 4758 switch (VA.getLocInfo()) { 4759 default: llvm_unreachable("Unknown loc info!"); 4760 case CCValAssign::Full: break; 4761 case CCValAssign::AExt: 4762 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 4763 break; 4764 case CCValAssign::ZExt: 4765 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 4766 break; 4767 case CCValAssign::SExt: 4768 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 4769 break; 4770 } 4771 4772 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 4773 Flag = Chain.getValue(1); 4774 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 4775 } 4776 4777 RetOps[0] = Chain; // Update chain. 4778 4779 // Add the flag if we have it. 4780 if (Flag.getNode()) 4781 RetOps.push_back(Flag); 4782 4783 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); 4784} 4785 4786SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG, 4787 const PPCSubtarget &Subtarget) const { 4788 // When we pop the dynamic allocation we need to restore the SP link. 4789 SDLoc dl(Op); 4790 4791 // Get the corect type for pointers. 4792 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 4793 4794 // Construct the stack pointer operand. 4795 bool isPPC64 = Subtarget.isPPC64(); 4796 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; 4797 SDValue StackPtr = DAG.getRegister(SP, PtrVT); 4798 4799 // Get the operands for the STACKRESTORE. 4800 SDValue Chain = Op.getOperand(0); 4801 SDValue SaveSP = Op.getOperand(1); 4802 4803 // Load the old link SP. 4804 SDValue LoadLinkSP = DAG.getLoad(PtrVT, dl, Chain, StackPtr, 4805 MachinePointerInfo(), 4806 false, false, false, 0); 4807 4808 // Restore the stack pointer. 4809 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); 4810 4811 // Store the old link SP. 4812 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo(), 4813 false, false, 0); 4814} 4815 4816 4817 4818SDValue 4819PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG & DAG) const { 4820 MachineFunction &MF = DAG.getMachineFunction(); 4821 bool isPPC64 = PPCSubTarget.isPPC64(); 4822 bool isDarwinABI = PPCSubTarget.isDarwinABI(); 4823 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 4824 4825 // Get current frame pointer save index. The users of this index will be 4826 // primarily DYNALLOC instructions. 4827 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 4828 int RASI = FI->getReturnAddrSaveIndex(); 4829 4830 // If the frame pointer save index hasn't been defined yet. 4831 if (!RASI) { 4832 // Find out what the fix offset of the frame pointer save area. 4833 int LROffset = PPCFrameLowering::getReturnSaveOffset(isPPC64, isDarwinABI); 4834 // Allocate the frame index for frame pointer save area. 4835 RASI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, LROffset, true); 4836 // Save the result. 4837 FI->setReturnAddrSaveIndex(RASI); 4838 } 4839 return DAG.getFrameIndex(RASI, PtrVT); 4840} 4841 4842SDValue 4843PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { 4844 MachineFunction &MF = DAG.getMachineFunction(); 4845 bool isPPC64 = PPCSubTarget.isPPC64(); 4846 bool isDarwinABI = PPCSubTarget.isDarwinABI(); 4847 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 4848 4849 // Get current frame pointer save index. The users of this index will be 4850 // primarily DYNALLOC instructions. 4851 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 4852 int FPSI = FI->getFramePointerSaveIndex(); 4853 4854 // If the frame pointer save index hasn't been defined yet. 4855 if (!FPSI) { 4856 // Find out what the fix offset of the frame pointer save area. 4857 int FPOffset = PPCFrameLowering::getFramePointerSaveOffset(isPPC64, 4858 isDarwinABI); 4859 4860 // Allocate the frame index for frame pointer save area. 4861 FPSI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); 4862 // Save the result. 4863 FI->setFramePointerSaveIndex(FPSI); 4864 } 4865 return DAG.getFrameIndex(FPSI, PtrVT); 4866} 4867 4868SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 4869 SelectionDAG &DAG, 4870 const PPCSubtarget &Subtarget) const { 4871 // Get the inputs. 4872 SDValue Chain = Op.getOperand(0); 4873 SDValue Size = Op.getOperand(1); 4874 SDLoc dl(Op); 4875 4876 // Get the corect type for pointers. 4877 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 4878 // Negate the size. 4879 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, 4880 DAG.getConstant(0, PtrVT), Size); 4881 // Construct a node for the frame pointer save index. 4882 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 4883 // Build a DYNALLOC node. 4884 SDValue Ops[3] = { Chain, NegSize, FPSIdx }; 4885 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); 4886 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); 4887} 4888 4889SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, 4890 SelectionDAG &DAG) const { 4891 SDLoc DL(Op); 4892 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, 4893 DAG.getVTList(MVT::i32, MVT::Other), 4894 Op.getOperand(0), Op.getOperand(1)); 4895} 4896 4897SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, 4898 SelectionDAG &DAG) const { 4899 SDLoc DL(Op); 4900 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, 4901 Op.getOperand(0), Op.getOperand(1)); 4902} 4903 4904SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 4905 assert(Op.getValueType() == MVT::i1 && 4906 "Custom lowering only for i1 loads"); 4907 4908 // First, load 8 bits into 32 bits, then truncate to 1 bit. 4909 4910 SDLoc dl(Op); 4911 LoadSDNode *LD = cast<LoadSDNode>(Op); 4912 4913 SDValue Chain = LD->getChain(); 4914 SDValue BasePtr = LD->getBasePtr(); 4915 MachineMemOperand *MMO = LD->getMemOperand(); 4916 4917 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(), Chain, 4918 BasePtr, MVT::i8, MMO); 4919 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); 4920 4921 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; 4922 return DAG.getMergeValues(Ops, dl); 4923} 4924 4925SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 4926 assert(Op.getOperand(1).getValueType() == MVT::i1 && 4927 "Custom lowering only for i1 stores"); 4928 4929 // First, zero extend to 32 bits, then use a truncating store to 8 bits. 4930 4931 SDLoc dl(Op); 4932 StoreSDNode *ST = cast<StoreSDNode>(Op); 4933 4934 SDValue Chain = ST->getChain(); 4935 SDValue BasePtr = ST->getBasePtr(); 4936 SDValue Value = ST->getValue(); 4937 MachineMemOperand *MMO = ST->getMemOperand(); 4938 4939 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(), Value); 4940 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); 4941} 4942 4943// FIXME: Remove this once the ANDI glue bug is fixed: 4944SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { 4945 assert(Op.getValueType() == MVT::i1 && 4946 "Custom lowering only for i1 results"); 4947 4948 SDLoc DL(Op); 4949 return DAG.getNode(PPCISD::ANDIo_1_GT_BIT, DL, MVT::i1, 4950 Op.getOperand(0)); 4951} 4952 4953/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when 4954/// possible. 4955SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 4956 // Not FP? Not a fsel. 4957 if (!Op.getOperand(0).getValueType().isFloatingPoint() || 4958 !Op.getOperand(2).getValueType().isFloatingPoint()) 4959 return Op; 4960 4961 // We might be able to do better than this under some circumstances, but in 4962 // general, fsel-based lowering of select is a finite-math-only optimization. 4963 // For more information, see section F.3 of the 2.06 ISA specification. 4964 if (!DAG.getTarget().Options.NoInfsFPMath || 4965 !DAG.getTarget().Options.NoNaNsFPMath) 4966 return Op; 4967 4968 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 4969 4970 EVT ResVT = Op.getValueType(); 4971 EVT CmpVT = Op.getOperand(0).getValueType(); 4972 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 4973 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); 4974 SDLoc dl(Op); 4975 4976 // If the RHS of the comparison is a 0.0, we don't need to do the 4977 // subtraction at all. 4978 SDValue Sel1; 4979 if (isFloatingPointZero(RHS)) 4980 switch (CC) { 4981 default: break; // SETUO etc aren't handled by fsel. 4982 case ISD::SETNE: 4983 std::swap(TV, FV); 4984 case ISD::SETEQ: 4985 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 4986 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 4987 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 4988 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 4989 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 4990 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 4991 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); 4992 case ISD::SETULT: 4993 case ISD::SETLT: 4994 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 4995 case ISD::SETOGE: 4996 case ISD::SETGE: 4997 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 4998 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 4999 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 5000 case ISD::SETUGT: 5001 case ISD::SETGT: 5002 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 5003 case ISD::SETOLE: 5004 case ISD::SETLE: 5005 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 5006 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 5007 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 5008 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); 5009 } 5010 5011 SDValue Cmp; 5012 switch (CC) { 5013 default: break; // SETUO etc aren't handled by fsel. 5014 case ISD::SETNE: 5015 std::swap(TV, FV); 5016 case ISD::SETEQ: 5017 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS); 5018 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 5019 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 5020 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 5021 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 5022 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 5023 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 5024 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); 5025 case ISD::SETULT: 5026 case ISD::SETLT: 5027 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS); 5028 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 5029 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 5030 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 5031 case ISD::SETOGE: 5032 case ISD::SETGE: 5033 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS); 5034 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 5035 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 5036 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 5037 case ISD::SETUGT: 5038 case ISD::SETGT: 5039 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS); 5040 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 5041 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 5042 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 5043 case ISD::SETOLE: 5044 case ISD::SETLE: 5045 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS); 5046 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 5047 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 5048 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 5049 } 5050 return Op; 5051} 5052 5053// FIXME: Split this code up when LegalizeDAGTypes lands. 5054SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, 5055 SDLoc dl) const { 5056 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 5057 SDValue Src = Op.getOperand(0); 5058 if (Src.getValueType() == MVT::f32) 5059 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 5060 5061 SDValue Tmp; 5062 switch (Op.getSimpleValueType().SimpleTy) { 5063 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 5064 case MVT::i32: 5065 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIWZ : 5066 (PPCSubTarget.hasFPCVT() ? PPCISD::FCTIWUZ : 5067 PPCISD::FCTIDZ), 5068 dl, MVT::f64, Src); 5069 break; 5070 case MVT::i64: 5071 assert((Op.getOpcode() == ISD::FP_TO_SINT || PPCSubTarget.hasFPCVT()) && 5072 "i64 FP_TO_UINT is supported only with FPCVT"); 5073 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 5074 PPCISD::FCTIDUZ, 5075 dl, MVT::f64, Src); 5076 break; 5077 } 5078 5079 // Convert the FP value to an int value through memory. 5080 bool i32Stack = Op.getValueType() == MVT::i32 && PPCSubTarget.hasSTFIWX() && 5081 (Op.getOpcode() == ISD::FP_TO_SINT || PPCSubTarget.hasFPCVT()); 5082 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); 5083 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); 5084 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(FI); 5085 5086 // Emit a store to the stack slot. 5087 SDValue Chain; 5088 if (i32Stack) { 5089 MachineFunction &MF = DAG.getMachineFunction(); 5090 MachineMemOperand *MMO = 5091 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); 5092 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; 5093 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 5094 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); 5095 } else 5096 Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, 5097 MPI, false, false, 0); 5098 5099 // Result is a load from the stack slot. If loading 4 bytes, make sure to 5100 // add in a bias. 5101 if (Op.getValueType() == MVT::i32 && !i32Stack) { 5102 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, 5103 DAG.getConstant(4, FIPtr.getValueType())); 5104 MPI = MachinePointerInfo(); 5105 } 5106 5107 return DAG.getLoad(Op.getValueType(), dl, Chain, FIPtr, MPI, 5108 false, false, false, 0); 5109} 5110 5111SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, 5112 SelectionDAG &DAG) const { 5113 SDLoc dl(Op); 5114 // Don't handle ppc_fp128 here; let it be lowered to a libcall. 5115 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 5116 return SDValue(); 5117 5118 if (Op.getOperand(0).getValueType() == MVT::i1) 5119 return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), 5120 DAG.getConstantFP(1.0, Op.getValueType()), 5121 DAG.getConstantFP(0.0, Op.getValueType())); 5122 5123 assert((Op.getOpcode() == ISD::SINT_TO_FP || PPCSubTarget.hasFPCVT()) && 5124 "UINT_TO_FP is supported only with FPCVT"); 5125 5126 // If we have FCFIDS, then use it when converting to single-precision. 5127 // Otherwise, convert to double-precision and then round. 5128 unsigned FCFOp = (PPCSubTarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? 5129 (Op.getOpcode() == ISD::UINT_TO_FP ? 5130 PPCISD::FCFIDUS : PPCISD::FCFIDS) : 5131 (Op.getOpcode() == ISD::UINT_TO_FP ? 5132 PPCISD::FCFIDU : PPCISD::FCFID); 5133 MVT FCFTy = (PPCSubTarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? 5134 MVT::f32 : MVT::f64; 5135 5136 if (Op.getOperand(0).getValueType() == MVT::i64) { 5137 SDValue SINT = Op.getOperand(0); 5138 // When converting to single-precision, we actually need to convert 5139 // to double-precision first and then round to single-precision. 5140 // To avoid double-rounding effects during that operation, we have 5141 // to prepare the input operand. Bits that might be truncated when 5142 // converting to double-precision are replaced by a bit that won't 5143 // be lost at this stage, but is below the single-precision rounding 5144 // position. 5145 // 5146 // However, if -enable-unsafe-fp-math is in effect, accept double 5147 // rounding to avoid the extra overhead. 5148 if (Op.getValueType() == MVT::f32 && 5149 !PPCSubTarget.hasFPCVT() && 5150 !DAG.getTarget().Options.UnsafeFPMath) { 5151 5152 // Twiddle input to make sure the low 11 bits are zero. (If this 5153 // is the case, we are guaranteed the value will fit into the 53 bit 5154 // mantissa of an IEEE double-precision value without rounding.) 5155 // If any of those low 11 bits were not zero originally, make sure 5156 // bit 12 (value 2048) is set instead, so that the final rounding 5157 // to single-precision gets the correct result. 5158 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, 5159 SINT, DAG.getConstant(2047, MVT::i64)); 5160 Round = DAG.getNode(ISD::ADD, dl, MVT::i64, 5161 Round, DAG.getConstant(2047, MVT::i64)); 5162 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); 5163 Round = DAG.getNode(ISD::AND, dl, MVT::i64, 5164 Round, DAG.getConstant(-2048, MVT::i64)); 5165 5166 // However, we cannot use that value unconditionally: if the magnitude 5167 // of the input value is small, the bit-twiddling we did above might 5168 // end up visibly changing the output. Fortunately, in that case, we 5169 // don't need to twiddle bits since the original input will convert 5170 // exactly to double-precision floating-point already. Therefore, 5171 // construct a conditional to use the original value if the top 11 5172 // bits are all sign-bit copies, and use the rounded value computed 5173 // above otherwise. 5174 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, 5175 SINT, DAG.getConstant(53, MVT::i32)); 5176 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, 5177 Cond, DAG.getConstant(1, MVT::i64)); 5178 Cond = DAG.getSetCC(dl, MVT::i32, 5179 Cond, DAG.getConstant(1, MVT::i64), ISD::SETUGT); 5180 5181 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); 5182 } 5183 5184 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); 5185 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); 5186 5187 if (Op.getValueType() == MVT::f32 && !PPCSubTarget.hasFPCVT()) 5188 FP = DAG.getNode(ISD::FP_ROUND, dl, 5189 MVT::f32, FP, DAG.getIntPtrConstant(0)); 5190 return FP; 5191 } 5192 5193 assert(Op.getOperand(0).getValueType() == MVT::i32 && 5194 "Unhandled INT_TO_FP type in custom expander!"); 5195 // Since we only generate this in 64-bit mode, we can take advantage of 5196 // 64-bit registers. In particular, sign extend the input value into the 5197 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack 5198 // then lfd it and fcfid it. 5199 MachineFunction &MF = DAG.getMachineFunction(); 5200 MachineFrameInfo *FrameInfo = MF.getFrameInfo(); 5201 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 5202 5203 SDValue Ld; 5204 if (PPCSubTarget.hasLFIWAX() || PPCSubTarget.hasFPCVT()) { 5205 int FrameIdx = FrameInfo->CreateStackObject(4, 4, false); 5206 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 5207 5208 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 5209 MachinePointerInfo::getFixedStack(FrameIdx), 5210 false, false, 0); 5211 5212 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 5213 "Expected an i32 store"); 5214 MachineMemOperand *MMO = 5215 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FrameIdx), 5216 MachineMemOperand::MOLoad, 4, 4); 5217 SDValue Ops[] = { Store, FIdx }; 5218 Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? 5219 PPCISD::LFIWZX : PPCISD::LFIWAX, 5220 dl, DAG.getVTList(MVT::f64, MVT::Other), 5221 Ops, MVT::i32, MMO); 5222 } else { 5223 assert(PPCSubTarget.isPPC64() && 5224 "i32->FP without LFIWAX supported only on PPC64"); 5225 5226 int FrameIdx = FrameInfo->CreateStackObject(8, 8, false); 5227 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 5228 5229 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, 5230 Op.getOperand(0)); 5231 5232 // STD the extended value into the stack slot. 5233 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Ext64, FIdx, 5234 MachinePointerInfo::getFixedStack(FrameIdx), 5235 false, false, 0); 5236 5237 // Load the value as a double. 5238 Ld = DAG.getLoad(MVT::f64, dl, Store, FIdx, 5239 MachinePointerInfo::getFixedStack(FrameIdx), 5240 false, false, false, 0); 5241 } 5242 5243 // FCFID it and return it. 5244 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); 5245 if (Op.getValueType() == MVT::f32 && !PPCSubTarget.hasFPCVT()) 5246 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0)); 5247 return FP; 5248} 5249 5250SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 5251 SelectionDAG &DAG) const { 5252 SDLoc dl(Op); 5253 /* 5254 The rounding mode is in bits 30:31 of FPSR, and has the following 5255 settings: 5256 00 Round to nearest 5257 01 Round to 0 5258 10 Round to +inf 5259 11 Round to -inf 5260 5261 FLT_ROUNDS, on the other hand, expects the following: 5262 -1 Undefined 5263 0 Round to 0 5264 1 Round to nearest 5265 2 Round to +inf 5266 3 Round to -inf 5267 5268 To perform the conversion, we do: 5269 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) 5270 */ 5271 5272 MachineFunction &MF = DAG.getMachineFunction(); 5273 EVT VT = Op.getValueType(); 5274 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 5275 5276 // Save FP Control Word to register 5277 EVT NodeTys[] = { 5278 MVT::f64, // return register 5279 MVT::Glue // unused in this context 5280 }; 5281 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None); 5282 5283 // Save FP register to stack slot 5284 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8, false); 5285 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); 5286 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, 5287 StackSlot, MachinePointerInfo(), false, false,0); 5288 5289 // Load FP Control Word from low 32 bits of stack slot. 5290 SDValue Four = DAG.getConstant(4, PtrVT); 5291 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); 5292 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo(), 5293 false, false, false, 0); 5294 5295 // Transform as necessary 5296 SDValue CWD1 = 5297 DAG.getNode(ISD::AND, dl, MVT::i32, 5298 CWD, DAG.getConstant(3, MVT::i32)); 5299 SDValue CWD2 = 5300 DAG.getNode(ISD::SRL, dl, MVT::i32, 5301 DAG.getNode(ISD::AND, dl, MVT::i32, 5302 DAG.getNode(ISD::XOR, dl, MVT::i32, 5303 CWD, DAG.getConstant(3, MVT::i32)), 5304 DAG.getConstant(3, MVT::i32)), 5305 DAG.getConstant(1, MVT::i32)); 5306 5307 SDValue RetVal = 5308 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); 5309 5310 return DAG.getNode((VT.getSizeInBits() < 16 ? 5311 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); 5312} 5313 5314SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { 5315 EVT VT = Op.getValueType(); 5316 unsigned BitWidth = VT.getSizeInBits(); 5317 SDLoc dl(Op); 5318 assert(Op.getNumOperands() == 3 && 5319 VT == Op.getOperand(1).getValueType() && 5320 "Unexpected SHL!"); 5321 5322 // Expand into a bunch of logical ops. Note that these ops 5323 // depend on the PPC behavior for oversized shift amounts. 5324 SDValue Lo = Op.getOperand(0); 5325 SDValue Hi = Op.getOperand(1); 5326 SDValue Amt = Op.getOperand(2); 5327 EVT AmtVT = Amt.getValueType(); 5328 5329 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 5330 DAG.getConstant(BitWidth, AmtVT), Amt); 5331 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); 5332 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); 5333 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); 5334 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 5335 DAG.getConstant(-BitWidth, AmtVT)); 5336 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); 5337 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 5338 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); 5339 SDValue OutOps[] = { OutLo, OutHi }; 5340 return DAG.getMergeValues(OutOps, dl); 5341} 5342 5343SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { 5344 EVT VT = Op.getValueType(); 5345 SDLoc dl(Op); 5346 unsigned BitWidth = VT.getSizeInBits(); 5347 assert(Op.getNumOperands() == 3 && 5348 VT == Op.getOperand(1).getValueType() && 5349 "Unexpected SRL!"); 5350 5351 // Expand into a bunch of logical ops. Note that these ops 5352 // depend on the PPC behavior for oversized shift amounts. 5353 SDValue Lo = Op.getOperand(0); 5354 SDValue Hi = Op.getOperand(1); 5355 SDValue Amt = Op.getOperand(2); 5356 EVT AmtVT = Amt.getValueType(); 5357 5358 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 5359 DAG.getConstant(BitWidth, AmtVT), Amt); 5360 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 5361 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 5362 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 5363 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 5364 DAG.getConstant(-BitWidth, AmtVT)); 5365 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); 5366 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 5367 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); 5368 SDValue OutOps[] = { OutLo, OutHi }; 5369 return DAG.getMergeValues(OutOps, dl); 5370} 5371 5372SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { 5373 SDLoc dl(Op); 5374 EVT VT = Op.getValueType(); 5375 unsigned BitWidth = VT.getSizeInBits(); 5376 assert(Op.getNumOperands() == 3 && 5377 VT == Op.getOperand(1).getValueType() && 5378 "Unexpected SRA!"); 5379 5380 // Expand into a bunch of logical ops, followed by a select_cc. 5381 SDValue Lo = Op.getOperand(0); 5382 SDValue Hi = Op.getOperand(1); 5383 SDValue Amt = Op.getOperand(2); 5384 EVT AmtVT = Amt.getValueType(); 5385 5386 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 5387 DAG.getConstant(BitWidth, AmtVT), Amt); 5388 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 5389 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 5390 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 5391 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 5392 DAG.getConstant(-BitWidth, AmtVT)); 5393 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); 5394 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); 5395 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, AmtVT), 5396 Tmp4, Tmp6, ISD::SETLE); 5397 SDValue OutOps[] = { OutLo, OutHi }; 5398 return DAG.getMergeValues(OutOps, dl); 5399} 5400 5401//===----------------------------------------------------------------------===// 5402// Vector related lowering. 5403// 5404 5405/// BuildSplatI - Build a canonical splati of Val with an element size of 5406/// SplatSize. Cast the result to VT. 5407static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, 5408 SelectionDAG &DAG, SDLoc dl) { 5409 assert(Val >= -16 && Val <= 15 && "vsplti is out of range!"); 5410 5411 static const EVT VTys[] = { // canonical VT to use for each size. 5412 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 5413 }; 5414 5415 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; 5416 5417 // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. 5418 if (Val == -1) 5419 SplatSize = 1; 5420 5421 EVT CanonicalVT = VTys[SplatSize-1]; 5422 5423 // Build a canonical splat for this value. 5424 SDValue Elt = DAG.getConstant(Val, MVT::i32); 5425 SmallVector<SDValue, 8> Ops; 5426 Ops.assign(CanonicalVT.getVectorNumElements(), Elt); 5427 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, CanonicalVT, Ops); 5428 return DAG.getNode(ISD::BITCAST, dl, ReqVT, Res); 5429} 5430 5431/// BuildIntrinsicOp - Return a unary operator intrinsic node with the 5432/// specified intrinsic ID. 5433static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, 5434 SelectionDAG &DAG, SDLoc dl, 5435 EVT DestVT = MVT::Other) { 5436 if (DestVT == MVT::Other) DestVT = Op.getValueType(); 5437 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 5438 DAG.getConstant(IID, MVT::i32), Op); 5439} 5440 5441/// BuildIntrinsicOp - Return a binary operator intrinsic node with the 5442/// specified intrinsic ID. 5443static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, 5444 SelectionDAG &DAG, SDLoc dl, 5445 EVT DestVT = MVT::Other) { 5446 if (DestVT == MVT::Other) DestVT = LHS.getValueType(); 5447 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 5448 DAG.getConstant(IID, MVT::i32), LHS, RHS); 5449} 5450 5451/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the 5452/// specified intrinsic ID. 5453static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, 5454 SDValue Op2, SelectionDAG &DAG, 5455 SDLoc dl, EVT DestVT = MVT::Other) { 5456 if (DestVT == MVT::Other) DestVT = Op0.getValueType(); 5457 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 5458 DAG.getConstant(IID, MVT::i32), Op0, Op1, Op2); 5459} 5460 5461 5462/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified 5463/// amount. The result has the specified value type. 5464static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, 5465 EVT VT, SelectionDAG &DAG, SDLoc dl) { 5466 // Force LHS/RHS to be the right type. 5467 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); 5468 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); 5469 5470 int Ops[16]; 5471 for (unsigned i = 0; i != 16; ++i) 5472 Ops[i] = i + Amt; 5473 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); 5474 return DAG.getNode(ISD::BITCAST, dl, VT, T); 5475} 5476 5477// If this is a case we can't handle, return null and let the default 5478// expansion code take care of it. If we CAN select this case, and if it 5479// selects to a single instruction, return Op. Otherwise, if we can codegen 5480// this case more efficiently than a constant pool load, lower it to the 5481// sequence of ops that should be used. 5482SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, 5483 SelectionDAG &DAG) const { 5484 SDLoc dl(Op); 5485 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 5486 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); 5487 5488 // Check if this is a splat of a constant value. 5489 APInt APSplatBits, APSplatUndef; 5490 unsigned SplatBitSize; 5491 bool HasAnyUndefs; 5492 if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, 5493 HasAnyUndefs, 0, true) || SplatBitSize > 32) 5494 return SDValue(); 5495 5496 unsigned SplatBits = APSplatBits.getZExtValue(); 5497 unsigned SplatUndef = APSplatUndef.getZExtValue(); 5498 unsigned SplatSize = SplatBitSize / 8; 5499 5500 // First, handle single instruction cases. 5501 5502 // All zeros? 5503 if (SplatBits == 0) { 5504 // Canonicalize all zero vectors to be v4i32. 5505 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { 5506 SDValue Z = DAG.getConstant(0, MVT::i32); 5507 Z = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Z, Z, Z, Z); 5508 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); 5509 } 5510 return Op; 5511 } 5512 5513 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. 5514 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> 5515 (32-SplatBitSize)); 5516 if (SextVal >= -16 && SextVal <= 15) 5517 return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); 5518 5519 5520 // Two instruction sequences. 5521 5522 // If this value is in the range [-32,30] and is even, use: 5523 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) 5524 // If this value is in the range [17,31] and is odd, use: 5525 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) 5526 // If this value is in the range [-31,-17] and is odd, use: 5527 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) 5528 // Note the last two are three-instruction sequences. 5529 if (SextVal >= -32 && SextVal <= 31) { 5530 // To avoid having these optimizations undone by constant folding, 5531 // we convert to a pseudo that will be expanded later into one of 5532 // the above forms. 5533 SDValue Elt = DAG.getConstant(SextVal, MVT::i32); 5534 EVT VT = (SplatSize == 1 ? MVT::v16i8 : 5535 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); 5536 SDValue EltSize = DAG.getConstant(SplatSize, MVT::i32); 5537 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); 5538 if (VT == Op.getValueType()) 5539 return RetVal; 5540 else 5541 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); 5542 } 5543 5544 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is 5545 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important 5546 // for fneg/fabs. 5547 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { 5548 // Make -1 and vspltisw -1: 5549 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); 5550 5551 // Make the VSLW intrinsic, computing 0x8000_0000. 5552 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, 5553 OnesV, DAG, dl); 5554 5555 // xor by OnesV to invert it. 5556 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); 5557 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 5558 } 5559 5560 // Check to see if this is a wide variety of vsplti*, binop self cases. 5561 static const signed char SplatCsts[] = { 5562 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, 5563 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 5564 }; 5565 5566 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { 5567 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for 5568 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' 5569 int i = SplatCsts[idx]; 5570 5571 // Figure out what shift amount will be used by altivec if shifted by i in 5572 // this splat size. 5573 unsigned TypeShiftAmt = i & (SplatBitSize-1); 5574 5575 // vsplti + shl self. 5576 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { 5577 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 5578 static const unsigned IIDs[] = { // Intrinsic to use for each size. 5579 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, 5580 Intrinsic::ppc_altivec_vslw 5581 }; 5582 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 5583 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 5584 } 5585 5586 // vsplti + srl self. 5587 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 5588 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 5589 static const unsigned IIDs[] = { // Intrinsic to use for each size. 5590 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, 5591 Intrinsic::ppc_altivec_vsrw 5592 }; 5593 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 5594 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 5595 } 5596 5597 // vsplti + sra self. 5598 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 5599 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 5600 static const unsigned IIDs[] = { // Intrinsic to use for each size. 5601 Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, 5602 Intrinsic::ppc_altivec_vsraw 5603 }; 5604 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 5605 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 5606 } 5607 5608 // vsplti + rol self. 5609 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | 5610 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { 5611 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 5612 static const unsigned IIDs[] = { // Intrinsic to use for each size. 5613 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, 5614 Intrinsic::ppc_altivec_vrlw 5615 }; 5616 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 5617 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 5618 } 5619 5620 // t = vsplti c, result = vsldoi t, t, 1 5621 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { 5622 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 5623 return BuildVSLDOI(T, T, 1, Op.getValueType(), DAG, dl); 5624 } 5625 // t = vsplti c, result = vsldoi t, t, 2 5626 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { 5627 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 5628 return BuildVSLDOI(T, T, 2, Op.getValueType(), DAG, dl); 5629 } 5630 // t = vsplti c, result = vsldoi t, t, 3 5631 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { 5632 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 5633 return BuildVSLDOI(T, T, 3, Op.getValueType(), DAG, dl); 5634 } 5635 } 5636 5637 return SDValue(); 5638} 5639 5640/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 5641/// the specified operations to build the shuffle. 5642static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 5643 SDValue RHS, SelectionDAG &DAG, 5644 SDLoc dl) { 5645 unsigned OpNum = (PFEntry >> 26) & 0x0F; 5646 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 5647 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 5648 5649 enum { 5650 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 5651 OP_VMRGHW, 5652 OP_VMRGLW, 5653 OP_VSPLTISW0, 5654 OP_VSPLTISW1, 5655 OP_VSPLTISW2, 5656 OP_VSPLTISW3, 5657 OP_VSLDOI4, 5658 OP_VSLDOI8, 5659 OP_VSLDOI12 5660 }; 5661 5662 if (OpNum == OP_COPY) { 5663 if (LHSID == (1*9+2)*9+3) return LHS; 5664 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 5665 return RHS; 5666 } 5667 5668 SDValue OpLHS, OpRHS; 5669 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 5670 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 5671 5672 int ShufIdxs[16]; 5673 switch (OpNum) { 5674 default: llvm_unreachable("Unknown i32 permute!"); 5675 case OP_VMRGHW: 5676 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; 5677 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; 5678 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; 5679 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; 5680 break; 5681 case OP_VMRGLW: 5682 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; 5683 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; 5684 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; 5685 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; 5686 break; 5687 case OP_VSPLTISW0: 5688 for (unsigned i = 0; i != 16; ++i) 5689 ShufIdxs[i] = (i&3)+0; 5690 break; 5691 case OP_VSPLTISW1: 5692 for (unsigned i = 0; i != 16; ++i) 5693 ShufIdxs[i] = (i&3)+4; 5694 break; 5695 case OP_VSPLTISW2: 5696 for (unsigned i = 0; i != 16; ++i) 5697 ShufIdxs[i] = (i&3)+8; 5698 break; 5699 case OP_VSPLTISW3: 5700 for (unsigned i = 0; i != 16; ++i) 5701 ShufIdxs[i] = (i&3)+12; 5702 break; 5703 case OP_VSLDOI4: 5704 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); 5705 case OP_VSLDOI8: 5706 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); 5707 case OP_VSLDOI12: 5708 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); 5709 } 5710 EVT VT = OpLHS.getValueType(); 5711 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); 5712 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); 5713 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); 5714 return DAG.getNode(ISD::BITCAST, dl, VT, T); 5715} 5716 5717/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this 5718/// is a shuffle we can handle in a single instruction, return it. Otherwise, 5719/// return the code it can be lowered into. Worst case, it can always be 5720/// lowered into a vperm. 5721SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 5722 SelectionDAG &DAG) const { 5723 SDLoc dl(Op); 5724 SDValue V1 = Op.getOperand(0); 5725 SDValue V2 = Op.getOperand(1); 5726 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 5727 EVT VT = Op.getValueType(); 5728 5729 // Cases that are handled by instructions that take permute immediates 5730 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be 5731 // selected by the instruction selector. 5732 if (V2.getOpcode() == ISD::UNDEF) { 5733 if (PPC::isSplatShuffleMask(SVOp, 1) || 5734 PPC::isSplatShuffleMask(SVOp, 2) || 5735 PPC::isSplatShuffleMask(SVOp, 4) || 5736 PPC::isVPKUWUMShuffleMask(SVOp, true) || 5737 PPC::isVPKUHUMShuffleMask(SVOp, true) || 5738 PPC::isVSLDOIShuffleMask(SVOp, true) != -1 || 5739 PPC::isVMRGLShuffleMask(SVOp, 1, true) || 5740 PPC::isVMRGLShuffleMask(SVOp, 2, true) || 5741 PPC::isVMRGLShuffleMask(SVOp, 4, true) || 5742 PPC::isVMRGHShuffleMask(SVOp, 1, true) || 5743 PPC::isVMRGHShuffleMask(SVOp, 2, true) || 5744 PPC::isVMRGHShuffleMask(SVOp, 4, true)) { 5745 return Op; 5746 } 5747 } 5748 5749 // Altivec has a variety of "shuffle immediates" that take two vector inputs 5750 // and produce a fixed permutation. If any of these match, do not lower to 5751 // VPERM. 5752 if (PPC::isVPKUWUMShuffleMask(SVOp, false) || 5753 PPC::isVPKUHUMShuffleMask(SVOp, false) || 5754 PPC::isVSLDOIShuffleMask(SVOp, false) != -1 || 5755 PPC::isVMRGLShuffleMask(SVOp, 1, false) || 5756 PPC::isVMRGLShuffleMask(SVOp, 2, false) || 5757 PPC::isVMRGLShuffleMask(SVOp, 4, false) || 5758 PPC::isVMRGHShuffleMask(SVOp, 1, false) || 5759 PPC::isVMRGHShuffleMask(SVOp, 2, false) || 5760 PPC::isVMRGHShuffleMask(SVOp, 4, false)) 5761 return Op; 5762 5763 // Check to see if this is a shuffle of 4-byte values. If so, we can use our 5764 // perfect shuffle table to emit an optimal matching sequence. 5765 ArrayRef<int> PermMask = SVOp->getMask(); 5766 5767 unsigned PFIndexes[4]; 5768 bool isFourElementShuffle = true; 5769 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number 5770 unsigned EltNo = 8; // Start out undef. 5771 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. 5772 if (PermMask[i*4+j] < 0) 5773 continue; // Undef, ignore it. 5774 5775 unsigned ByteSource = PermMask[i*4+j]; 5776 if ((ByteSource & 3) != j) { 5777 isFourElementShuffle = false; 5778 break; 5779 } 5780 5781 if (EltNo == 8) { 5782 EltNo = ByteSource/4; 5783 } else if (EltNo != ByteSource/4) { 5784 isFourElementShuffle = false; 5785 break; 5786 } 5787 } 5788 PFIndexes[i] = EltNo; 5789 } 5790 5791 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the 5792 // perfect shuffle vector to determine if it is cost effective to do this as 5793 // discrete instructions, or whether we should use a vperm. 5794 if (isFourElementShuffle) { 5795 // Compute the index in the perfect shuffle table. 5796 unsigned PFTableIndex = 5797 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 5798 5799 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 5800 unsigned Cost = (PFEntry >> 30); 5801 5802 // Determining when to avoid vperm is tricky. Many things affect the cost 5803 // of vperm, particularly how many times the perm mask needs to be computed. 5804 // For example, if the perm mask can be hoisted out of a loop or is already 5805 // used (perhaps because there are multiple permutes with the same shuffle 5806 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of 5807 // the loop requires an extra register. 5808 // 5809 // As a compromise, we only emit discrete instructions if the shuffle can be 5810 // generated in 3 or fewer operations. When we have loop information 5811 // available, if this block is within a loop, we should avoid using vperm 5812 // for 3-operation perms and use a constant pool load instead. 5813 if (Cost < 3) 5814 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 5815 } 5816 5817 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant 5818 // vector that will get spilled to the constant pool. 5819 if (V2.getOpcode() == ISD::UNDEF) V2 = V1; 5820 5821 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except 5822 // that it is in input element units, not in bytes. Convert now. 5823 EVT EltVT = V1.getValueType().getVectorElementType(); 5824 unsigned BytesPerElement = EltVT.getSizeInBits()/8; 5825 5826 SmallVector<SDValue, 16> ResultMask; 5827 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 5828 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; 5829 5830 for (unsigned j = 0; j != BytesPerElement; ++j) 5831 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j, 5832 MVT::i32)); 5833 } 5834 5835 SDValue VPermMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8, 5836 ResultMask); 5837 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), V1, V2, VPermMask); 5838} 5839 5840/// getAltivecCompareInfo - Given an intrinsic, return false if it is not an 5841/// altivec comparison. If it is, return true and fill in Opc/isDot with 5842/// information about the intrinsic. 5843static bool getAltivecCompareInfo(SDValue Intrin, int &CompareOpc, 5844 bool &isDot) { 5845 unsigned IntrinsicID = 5846 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue(); 5847 CompareOpc = -1; 5848 isDot = false; 5849 switch (IntrinsicID) { 5850 default: return false; 5851 // Comparison predicates. 5852 case Intrinsic::ppc_altivec_vcmpbfp_p: CompareOpc = 966; isDot = 1; break; 5853 case Intrinsic::ppc_altivec_vcmpeqfp_p: CompareOpc = 198; isDot = 1; break; 5854 case Intrinsic::ppc_altivec_vcmpequb_p: CompareOpc = 6; isDot = 1; break; 5855 case Intrinsic::ppc_altivec_vcmpequh_p: CompareOpc = 70; isDot = 1; break; 5856 case Intrinsic::ppc_altivec_vcmpequw_p: CompareOpc = 134; isDot = 1; break; 5857 case Intrinsic::ppc_altivec_vcmpgefp_p: CompareOpc = 454; isDot = 1; break; 5858 case Intrinsic::ppc_altivec_vcmpgtfp_p: CompareOpc = 710; isDot = 1; break; 5859 case Intrinsic::ppc_altivec_vcmpgtsb_p: CompareOpc = 774; isDot = 1; break; 5860 case Intrinsic::ppc_altivec_vcmpgtsh_p: CompareOpc = 838; isDot = 1; break; 5861 case Intrinsic::ppc_altivec_vcmpgtsw_p: CompareOpc = 902; isDot = 1; break; 5862 case Intrinsic::ppc_altivec_vcmpgtub_p: CompareOpc = 518; isDot = 1; break; 5863 case Intrinsic::ppc_altivec_vcmpgtuh_p: CompareOpc = 582; isDot = 1; break; 5864 case Intrinsic::ppc_altivec_vcmpgtuw_p: CompareOpc = 646; isDot = 1; break; 5865 5866 // Normal Comparisons. 5867 case Intrinsic::ppc_altivec_vcmpbfp: CompareOpc = 966; isDot = 0; break; 5868 case Intrinsic::ppc_altivec_vcmpeqfp: CompareOpc = 198; isDot = 0; break; 5869 case Intrinsic::ppc_altivec_vcmpequb: CompareOpc = 6; isDot = 0; break; 5870 case Intrinsic::ppc_altivec_vcmpequh: CompareOpc = 70; isDot = 0; break; 5871 case Intrinsic::ppc_altivec_vcmpequw: CompareOpc = 134; isDot = 0; break; 5872 case Intrinsic::ppc_altivec_vcmpgefp: CompareOpc = 454; isDot = 0; break; 5873 case Intrinsic::ppc_altivec_vcmpgtfp: CompareOpc = 710; isDot = 0; break; 5874 case Intrinsic::ppc_altivec_vcmpgtsb: CompareOpc = 774; isDot = 0; break; 5875 case Intrinsic::ppc_altivec_vcmpgtsh: CompareOpc = 838; isDot = 0; break; 5876 case Intrinsic::ppc_altivec_vcmpgtsw: CompareOpc = 902; isDot = 0; break; 5877 case Intrinsic::ppc_altivec_vcmpgtub: CompareOpc = 518; isDot = 0; break; 5878 case Intrinsic::ppc_altivec_vcmpgtuh: CompareOpc = 582; isDot = 0; break; 5879 case Intrinsic::ppc_altivec_vcmpgtuw: CompareOpc = 646; isDot = 0; break; 5880 } 5881 return true; 5882} 5883 5884/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom 5885/// lower, do it, otherwise return null. 5886SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 5887 SelectionDAG &DAG) const { 5888 // If this is a lowered altivec predicate compare, CompareOpc is set to the 5889 // opcode number of the comparison. 5890 SDLoc dl(Op); 5891 int CompareOpc; 5892 bool isDot; 5893 if (!getAltivecCompareInfo(Op, CompareOpc, isDot)) 5894 return SDValue(); // Don't custom lower most intrinsics. 5895 5896 // If this is a non-dot comparison, make the VCMP node and we are done. 5897 if (!isDot) { 5898 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), 5899 Op.getOperand(1), Op.getOperand(2), 5900 DAG.getConstant(CompareOpc, MVT::i32)); 5901 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); 5902 } 5903 5904 // Create the PPCISD altivec 'dot' comparison node. 5905 SDValue Ops[] = { 5906 Op.getOperand(2), // LHS 5907 Op.getOperand(3), // RHS 5908 DAG.getConstant(CompareOpc, MVT::i32) 5909 }; 5910 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; 5911 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 5912 5913 // Now that we have the comparison, emit a copy from the CR to a GPR. 5914 // This is flagged to the above dot comparison. 5915 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, 5916 DAG.getRegister(PPC::CR6, MVT::i32), 5917 CompNode.getValue(1)); 5918 5919 // Unpack the result based on how the target uses it. 5920 unsigned BitNo; // Bit # of CR6. 5921 bool InvertBit; // Invert result? 5922 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 5923 default: // Can't happen, don't crash on invalid number though. 5924 case 0: // Return the value of the EQ bit of CR6. 5925 BitNo = 0; InvertBit = false; 5926 break; 5927 case 1: // Return the inverted value of the EQ bit of CR6. 5928 BitNo = 0; InvertBit = true; 5929 break; 5930 case 2: // Return the value of the LT bit of CR6. 5931 BitNo = 2; InvertBit = false; 5932 break; 5933 case 3: // Return the inverted value of the LT bit of CR6. 5934 BitNo = 2; InvertBit = true; 5935 break; 5936 } 5937 5938 // Shift the bit into the low position. 5939 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, 5940 DAG.getConstant(8-(3-BitNo), MVT::i32)); 5941 // Isolate the bit. 5942 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, 5943 DAG.getConstant(1, MVT::i32)); 5944 5945 // If we are supposed to, toggle the bit. 5946 if (InvertBit) 5947 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, 5948 DAG.getConstant(1, MVT::i32)); 5949 return Flags; 5950} 5951 5952SDValue PPCTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 5953 SelectionDAG &DAG) const { 5954 SDLoc dl(Op); 5955 // For v2i64 (VSX), we can pattern patch the v2i32 case (using fp <-> int 5956 // instructions), but for smaller types, we need to first extend up to v2i32 5957 // before doing going farther. 5958 if (Op.getValueType() == MVT::v2i64) { 5959 EVT ExtVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 5960 if (ExtVT != MVT::v2i32) { 5961 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)); 5962 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, Op, 5963 DAG.getValueType(EVT::getVectorVT(*DAG.getContext(), 5964 ExtVT.getVectorElementType(), 4))); 5965 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Op); 5966 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v2i64, Op, 5967 DAG.getValueType(MVT::v2i32)); 5968 } 5969 5970 return Op; 5971 } 5972 5973 return SDValue(); 5974} 5975 5976SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, 5977 SelectionDAG &DAG) const { 5978 SDLoc dl(Op); 5979 // Create a stack slot that is 16-byte aligned. 5980 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 5981 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false); 5982 EVT PtrVT = getPointerTy(); 5983 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 5984 5985 // Store the input value into Value#0 of the stack slot. 5986 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, 5987 Op.getOperand(0), FIdx, MachinePointerInfo(), 5988 false, false, 0); 5989 // Load it out. 5990 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo(), 5991 false, false, false, 0); 5992} 5993 5994SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 5995 SDLoc dl(Op); 5996 if (Op.getValueType() == MVT::v4i32) { 5997 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 5998 5999 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); 6000 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. 6001 6002 SDValue RHSSwap = // = vrlw RHS, 16 6003 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); 6004 6005 // Shrinkify inputs to v8i16. 6006 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); 6007 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); 6008 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); 6009 6010 // Low parts multiplied together, generating 32-bit results (we ignore the 6011 // top parts). 6012 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, 6013 LHS, RHS, DAG, dl, MVT::v4i32); 6014 6015 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, 6016 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); 6017 // Shift the high parts up 16 bits. 6018 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, 6019 Neg16, DAG, dl); 6020 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); 6021 } else if (Op.getValueType() == MVT::v8i16) { 6022 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 6023 6024 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl); 6025 6026 return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm, 6027 LHS, RHS, Zero, DAG, dl); 6028 } else if (Op.getValueType() == MVT::v16i8) { 6029 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 6030 6031 // Multiply the even 8-bit parts, producing 16-bit sums. 6032 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, 6033 LHS, RHS, DAG, dl, MVT::v8i16); 6034 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); 6035 6036 // Multiply the odd 8-bit parts, producing 16-bit sums. 6037 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, 6038 LHS, RHS, DAG, dl, MVT::v8i16); 6039 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); 6040 6041 // Merge the results together. 6042 int Ops[16]; 6043 for (unsigned i = 0; i != 8; ++i) { 6044 Ops[i*2 ] = 2*i+1; 6045 Ops[i*2+1] = 2*i+1+16; 6046 } 6047 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); 6048 } else { 6049 llvm_unreachable("Unknown mul to lower!"); 6050 } 6051} 6052 6053/// LowerOperation - Provide custom lowering hooks for some operations. 6054/// 6055SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 6056 switch (Op.getOpcode()) { 6057 default: llvm_unreachable("Wasn't expecting to be able to lower this!"); 6058 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 6059 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 6060 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 6061 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 6062 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 6063 case ISD::SETCC: return LowerSETCC(Op, DAG); 6064 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 6065 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 6066 case ISD::VASTART: 6067 return LowerVASTART(Op, DAG, PPCSubTarget); 6068 6069 case ISD::VAARG: 6070 return LowerVAARG(Op, DAG, PPCSubTarget); 6071 6072 case ISD::VACOPY: 6073 return LowerVACOPY(Op, DAG, PPCSubTarget); 6074 6075 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG, PPCSubTarget); 6076 case ISD::DYNAMIC_STACKALLOC: 6077 return LowerDYNAMIC_STACKALLOC(Op, DAG, PPCSubTarget); 6078 6079 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); 6080 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); 6081 6082 case ISD::LOAD: return LowerLOAD(Op, DAG); 6083 case ISD::STORE: return LowerSTORE(Op, DAG); 6084 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); 6085 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 6086 case ISD::FP_TO_UINT: 6087 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, 6088 SDLoc(Op)); 6089 case ISD::UINT_TO_FP: 6090 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 6091 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 6092 6093 // Lower 64-bit shifts. 6094 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); 6095 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); 6096 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); 6097 6098 // Vector-related lowering. 6099 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 6100 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 6101 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 6102 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 6103 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 6104 case ISD::MUL: return LowerMUL(Op, DAG); 6105 6106 // For counter-based loop handling. 6107 case ISD::INTRINSIC_W_CHAIN: return SDValue(); 6108 6109 // Frame & Return address. 6110 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 6111 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 6112 } 6113} 6114 6115void PPCTargetLowering::ReplaceNodeResults(SDNode *N, 6116 SmallVectorImpl<SDValue>&Results, 6117 SelectionDAG &DAG) const { 6118 const TargetMachine &TM = getTargetMachine(); 6119 SDLoc dl(N); 6120 switch (N->getOpcode()) { 6121 default: 6122 llvm_unreachable("Do not know how to custom type legalize this operation!"); 6123 case ISD::INTRINSIC_W_CHAIN: { 6124 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 6125 Intrinsic::ppc_is_decremented_ctr_nonzero) 6126 break; 6127 6128 assert(N->getValueType(0) == MVT::i1 && 6129 "Unexpected result type for CTR decrement intrinsic"); 6130 EVT SVT = getSetCCResultType(*DAG.getContext(), N->getValueType(0)); 6131 SDVTList VTs = DAG.getVTList(SVT, MVT::Other); 6132 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), 6133 N->getOperand(1)); 6134 6135 Results.push_back(NewInt); 6136 Results.push_back(NewInt.getValue(1)); 6137 break; 6138 } 6139 case ISD::VAARG: { 6140 if (!TM.getSubtarget<PPCSubtarget>().isSVR4ABI() 6141 || TM.getSubtarget<PPCSubtarget>().isPPC64()) 6142 return; 6143 6144 EVT VT = N->getValueType(0); 6145 6146 if (VT == MVT::i64) { 6147 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG, PPCSubTarget); 6148 6149 Results.push_back(NewNode); 6150 Results.push_back(NewNode.getValue(1)); 6151 } 6152 return; 6153 } 6154 case ISD::FP_ROUND_INREG: { 6155 assert(N->getValueType(0) == MVT::ppcf128); 6156 assert(N->getOperand(0).getValueType() == MVT::ppcf128); 6157 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 6158 MVT::f64, N->getOperand(0), 6159 DAG.getIntPtrConstant(0)); 6160 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 6161 MVT::f64, N->getOperand(0), 6162 DAG.getIntPtrConstant(1)); 6163 6164 // Add the two halves of the long double in round-to-zero mode. 6165 SDValue FPreg = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); 6166 6167 // We know the low half is about to be thrown away, so just use something 6168 // convenient. 6169 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::ppcf128, 6170 FPreg, FPreg)); 6171 return; 6172 } 6173 case ISD::FP_TO_SINT: 6174 // LowerFP_TO_INT() can only handle f32 and f64. 6175 if (N->getOperand(0).getValueType() == MVT::ppcf128) 6176 return; 6177 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); 6178 return; 6179 } 6180} 6181 6182 6183//===----------------------------------------------------------------------===// 6184// Other Lowering Code 6185//===----------------------------------------------------------------------===// 6186 6187MachineBasicBlock * 6188PPCTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB, 6189 bool is64bit, unsigned BinOpcode) const { 6190 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 6191 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6192 6193 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6194 MachineFunction *F = BB->getParent(); 6195 MachineFunction::iterator It = BB; 6196 ++It; 6197 6198 unsigned dest = MI->getOperand(0).getReg(); 6199 unsigned ptrA = MI->getOperand(1).getReg(); 6200 unsigned ptrB = MI->getOperand(2).getReg(); 6201 unsigned incr = MI->getOperand(3).getReg(); 6202 DebugLoc dl = MI->getDebugLoc(); 6203 6204 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 6205 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 6206 F->insert(It, loopMBB); 6207 F->insert(It, exitMBB); 6208 exitMBB->splice(exitMBB->begin(), BB, 6209 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 6210 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 6211 6212 MachineRegisterInfo &RegInfo = F->getRegInfo(); 6213 unsigned TmpReg = (!BinOpcode) ? incr : 6214 RegInfo.createVirtualRegister( 6215 is64bit ? (const TargetRegisterClass *) &PPC::G8RCRegClass : 6216 (const TargetRegisterClass *) &PPC::GPRCRegClass); 6217 6218 // thisMBB: 6219 // ... 6220 // fallthrough --> loopMBB 6221 BB->addSuccessor(loopMBB); 6222 6223 // loopMBB: 6224 // l[wd]arx dest, ptr 6225 // add r0, dest, incr 6226 // st[wd]cx. r0, ptr 6227 // bne- loopMBB 6228 // fallthrough --> exitMBB 6229 BB = loopMBB; 6230 BuildMI(BB, dl, TII->get(is64bit ? PPC::LDARX : PPC::LWARX), dest) 6231 .addReg(ptrA).addReg(ptrB); 6232 if (BinOpcode) 6233 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); 6234 BuildMI(BB, dl, TII->get(is64bit ? PPC::STDCX : PPC::STWCX)) 6235 .addReg(TmpReg).addReg(ptrA).addReg(ptrB); 6236 BuildMI(BB, dl, TII->get(PPC::BCC)) 6237 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 6238 BB->addSuccessor(loopMBB); 6239 BB->addSuccessor(exitMBB); 6240 6241 // exitMBB: 6242 // ... 6243 BB = exitMBB; 6244 return BB; 6245} 6246 6247MachineBasicBlock * 6248PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr *MI, 6249 MachineBasicBlock *BB, 6250 bool is8bit, // operation 6251 unsigned BinOpcode) const { 6252 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 6253 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6254 // In 64 bit mode we have to use 64 bits for addresses, even though the 6255 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address 6256 // registers without caring whether they're 32 or 64, but here we're 6257 // doing actual arithmetic on the addresses. 6258 bool is64bit = PPCSubTarget.isPPC64(); 6259 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 6260 6261 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6262 MachineFunction *F = BB->getParent(); 6263 MachineFunction::iterator It = BB; 6264 ++It; 6265 6266 unsigned dest = MI->getOperand(0).getReg(); 6267 unsigned ptrA = MI->getOperand(1).getReg(); 6268 unsigned ptrB = MI->getOperand(2).getReg(); 6269 unsigned incr = MI->getOperand(3).getReg(); 6270 DebugLoc dl = MI->getDebugLoc(); 6271 6272 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 6273 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 6274 F->insert(It, loopMBB); 6275 F->insert(It, exitMBB); 6276 exitMBB->splice(exitMBB->begin(), BB, 6277 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 6278 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 6279 6280 MachineRegisterInfo &RegInfo = F->getRegInfo(); 6281 const TargetRegisterClass *RC = 6282 is64bit ? (const TargetRegisterClass *) &PPC::G8RCRegClass : 6283 (const TargetRegisterClass *) &PPC::GPRCRegClass; 6284 unsigned PtrReg = RegInfo.createVirtualRegister(RC); 6285 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC); 6286 unsigned ShiftReg = RegInfo.createVirtualRegister(RC); 6287 unsigned Incr2Reg = RegInfo.createVirtualRegister(RC); 6288 unsigned MaskReg = RegInfo.createVirtualRegister(RC); 6289 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC); 6290 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC); 6291 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC); 6292 unsigned Tmp3Reg = RegInfo.createVirtualRegister(RC); 6293 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC); 6294 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC); 6295 unsigned Ptr1Reg; 6296 unsigned TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RC); 6297 6298 // thisMBB: 6299 // ... 6300 // fallthrough --> loopMBB 6301 BB->addSuccessor(loopMBB); 6302 6303 // The 4-byte load must be aligned, while a char or short may be 6304 // anywhere in the word. Hence all this nasty bookkeeping code. 6305 // add ptr1, ptrA, ptrB [copy if ptrA==0] 6306 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 6307 // xori shift, shift1, 24 [16] 6308 // rlwinm ptr, ptr1, 0, 0, 29 6309 // slw incr2, incr, shift 6310 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 6311 // slw mask, mask2, shift 6312 // loopMBB: 6313 // lwarx tmpDest, ptr 6314 // add tmp, tmpDest, incr2 6315 // andc tmp2, tmpDest, mask 6316 // and tmp3, tmp, mask 6317 // or tmp4, tmp3, tmp2 6318 // stwcx. tmp4, ptr 6319 // bne- loopMBB 6320 // fallthrough --> exitMBB 6321 // srw dest, tmpDest, shift 6322 if (ptrA != ZeroReg) { 6323 Ptr1Reg = RegInfo.createVirtualRegister(RC); 6324 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 6325 .addReg(ptrA).addReg(ptrB); 6326 } else { 6327 Ptr1Reg = ptrB; 6328 } 6329 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg) 6330 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27); 6331 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg) 6332 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16); 6333 if (is64bit) 6334 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 6335 .addReg(Ptr1Reg).addImm(0).addImm(61); 6336 else 6337 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 6338 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29); 6339 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg) 6340 .addReg(incr).addReg(ShiftReg); 6341 if (is8bit) 6342 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 6343 else { 6344 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 6345 BuildMI(BB, dl, TII->get(PPC::ORI),Mask2Reg).addReg(Mask3Reg).addImm(65535); 6346 } 6347 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 6348 .addReg(Mask2Reg).addReg(ShiftReg); 6349 6350 BB = loopMBB; 6351 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 6352 .addReg(ZeroReg).addReg(PtrReg); 6353 if (BinOpcode) 6354 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) 6355 .addReg(Incr2Reg).addReg(TmpDestReg); 6356 BuildMI(BB, dl, TII->get(is64bit ? PPC::ANDC8 : PPC::ANDC), Tmp2Reg) 6357 .addReg(TmpDestReg).addReg(MaskReg); 6358 BuildMI(BB, dl, TII->get(is64bit ? PPC::AND8 : PPC::AND), Tmp3Reg) 6359 .addReg(TmpReg).addReg(MaskReg); 6360 BuildMI(BB, dl, TII->get(is64bit ? PPC::OR8 : PPC::OR), Tmp4Reg) 6361 .addReg(Tmp3Reg).addReg(Tmp2Reg); 6362 BuildMI(BB, dl, TII->get(PPC::STWCX)) 6363 .addReg(Tmp4Reg).addReg(ZeroReg).addReg(PtrReg); 6364 BuildMI(BB, dl, TII->get(PPC::BCC)) 6365 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 6366 BB->addSuccessor(loopMBB); 6367 BB->addSuccessor(exitMBB); 6368 6369 // exitMBB: 6370 // ... 6371 BB = exitMBB; 6372 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest).addReg(TmpDestReg) 6373 .addReg(ShiftReg); 6374 return BB; 6375} 6376 6377llvm::MachineBasicBlock* 6378PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr *MI, 6379 MachineBasicBlock *MBB) const { 6380 DebugLoc DL = MI->getDebugLoc(); 6381 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6382 6383 MachineFunction *MF = MBB->getParent(); 6384 MachineRegisterInfo &MRI = MF->getRegInfo(); 6385 6386 const BasicBlock *BB = MBB->getBasicBlock(); 6387 MachineFunction::iterator I = MBB; 6388 ++I; 6389 6390 // Memory Reference 6391 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 6392 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 6393 6394 unsigned DstReg = MI->getOperand(0).getReg(); 6395 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 6396 assert(RC->hasType(MVT::i32) && "Invalid destination!"); 6397 unsigned mainDstReg = MRI.createVirtualRegister(RC); 6398 unsigned restoreDstReg = MRI.createVirtualRegister(RC); 6399 6400 MVT PVT = getPointerTy(); 6401 assert((PVT == MVT::i64 || PVT == MVT::i32) && 6402 "Invalid Pointer Size!"); 6403 // For v = setjmp(buf), we generate 6404 // 6405 // thisMBB: 6406 // SjLjSetup mainMBB 6407 // bl mainMBB 6408 // v_restore = 1 6409 // b sinkMBB 6410 // 6411 // mainMBB: 6412 // buf[LabelOffset] = LR 6413 // v_main = 0 6414 // 6415 // sinkMBB: 6416 // v = phi(main, restore) 6417 // 6418 6419 MachineBasicBlock *thisMBB = MBB; 6420 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); 6421 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); 6422 MF->insert(I, mainMBB); 6423 MF->insert(I, sinkMBB); 6424 6425 MachineInstrBuilder MIB; 6426 6427 // Transfer the remainder of BB and its successor edges to sinkMBB. 6428 sinkMBB->splice(sinkMBB->begin(), MBB, 6429 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 6430 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); 6431 6432 // Note that the structure of the jmp_buf used here is not compatible 6433 // with that used by libc, and is not designed to be. Specifically, it 6434 // stores only those 'reserved' registers that LLVM does not otherwise 6435 // understand how to spill. Also, by convention, by the time this 6436 // intrinsic is called, Clang has already stored the frame address in the 6437 // first slot of the buffer and stack address in the third. Following the 6438 // X86 target code, we'll store the jump address in the second slot. We also 6439 // need to save the TOC pointer (R2) to handle jumps between shared 6440 // libraries, and that will be stored in the fourth slot. The thread 6441 // identifier (R13) is not affected. 6442 6443 // thisMBB: 6444 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 6445 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 6446 const int64_t BPOffset = 4 * PVT.getStoreSize(); 6447 6448 // Prepare IP either in reg. 6449 const TargetRegisterClass *PtrRC = getRegClassFor(PVT); 6450 unsigned LabelReg = MRI.createVirtualRegister(PtrRC); 6451 unsigned BufReg = MI->getOperand(1).getReg(); 6452 6453 if (PPCSubTarget.isPPC64() && PPCSubTarget.isSVR4ABI()) { 6454 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) 6455 .addReg(PPC::X2) 6456 .addImm(TOCOffset) 6457 .addReg(BufReg); 6458 MIB.setMemRefs(MMOBegin, MMOEnd); 6459 } 6460 6461 // Naked functions never have a base pointer, and so we use r1. For all 6462 // other functions, this decision must be delayed until during PEI. 6463 unsigned BaseReg; 6464 if (MF->getFunction()->getAttributes().hasAttribute( 6465 AttributeSet::FunctionIndex, Attribute::Naked)) 6466 BaseReg = PPCSubTarget.isPPC64() ? PPC::X1 : PPC::R1; 6467 else 6468 BaseReg = PPCSubTarget.isPPC64() ? PPC::BP8 : PPC::BP; 6469 6470 MIB = BuildMI(*thisMBB, MI, DL, 6471 TII->get(PPCSubTarget.isPPC64() ? PPC::STD : PPC::STW)) 6472 .addReg(BaseReg) 6473 .addImm(BPOffset) 6474 .addReg(BufReg); 6475 MIB.setMemRefs(MMOBegin, MMOEnd); 6476 6477 // Setup 6478 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); 6479 const PPCRegisterInfo *TRI = 6480 static_cast<const PPCRegisterInfo*>(getTargetMachine().getRegisterInfo()); 6481 MIB.addRegMask(TRI->getNoPreservedMask()); 6482 6483 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); 6484 6485 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) 6486 .addMBB(mainMBB); 6487 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); 6488 6489 thisMBB->addSuccessor(mainMBB, /* weight */ 0); 6490 thisMBB->addSuccessor(sinkMBB, /* weight */ 1); 6491 6492 // mainMBB: 6493 // mainDstReg = 0 6494 MIB = BuildMI(mainMBB, DL, 6495 TII->get(PPCSubTarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); 6496 6497 // Store IP 6498 if (PPCSubTarget.isPPC64()) { 6499 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) 6500 .addReg(LabelReg) 6501 .addImm(LabelOffset) 6502 .addReg(BufReg); 6503 } else { 6504 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) 6505 .addReg(LabelReg) 6506 .addImm(LabelOffset) 6507 .addReg(BufReg); 6508 } 6509 6510 MIB.setMemRefs(MMOBegin, MMOEnd); 6511 6512 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); 6513 mainMBB->addSuccessor(sinkMBB); 6514 6515 // sinkMBB: 6516 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 6517 TII->get(PPC::PHI), DstReg) 6518 .addReg(mainDstReg).addMBB(mainMBB) 6519 .addReg(restoreDstReg).addMBB(thisMBB); 6520 6521 MI->eraseFromParent(); 6522 return sinkMBB; 6523} 6524 6525MachineBasicBlock * 6526PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr *MI, 6527 MachineBasicBlock *MBB) const { 6528 DebugLoc DL = MI->getDebugLoc(); 6529 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6530 6531 MachineFunction *MF = MBB->getParent(); 6532 MachineRegisterInfo &MRI = MF->getRegInfo(); 6533 6534 // Memory Reference 6535 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin(); 6536 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end(); 6537 6538 MVT PVT = getPointerTy(); 6539 assert((PVT == MVT::i64 || PVT == MVT::i32) && 6540 "Invalid Pointer Size!"); 6541 6542 const TargetRegisterClass *RC = 6543 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 6544 unsigned Tmp = MRI.createVirtualRegister(RC); 6545 // Since FP is only updated here but NOT referenced, it's treated as GPR. 6546 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; 6547 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; 6548 unsigned BP = (PVT == MVT::i64) ? PPC::X30 : PPC::R30; 6549 6550 MachineInstrBuilder MIB; 6551 6552 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 6553 const int64_t SPOffset = 2 * PVT.getStoreSize(); 6554 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 6555 const int64_t BPOffset = 4 * PVT.getStoreSize(); 6556 6557 unsigned BufReg = MI->getOperand(0).getReg(); 6558 6559 // Reload FP (the jumped-to function may not have had a 6560 // frame pointer, and if so, then its r31 will be restored 6561 // as necessary). 6562 if (PVT == MVT::i64) { 6563 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) 6564 .addImm(0) 6565 .addReg(BufReg); 6566 } else { 6567 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) 6568 .addImm(0) 6569 .addReg(BufReg); 6570 } 6571 MIB.setMemRefs(MMOBegin, MMOEnd); 6572 6573 // Reload IP 6574 if (PVT == MVT::i64) { 6575 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) 6576 .addImm(LabelOffset) 6577 .addReg(BufReg); 6578 } else { 6579 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) 6580 .addImm(LabelOffset) 6581 .addReg(BufReg); 6582 } 6583 MIB.setMemRefs(MMOBegin, MMOEnd); 6584 6585 // Reload SP 6586 if (PVT == MVT::i64) { 6587 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) 6588 .addImm(SPOffset) 6589 .addReg(BufReg); 6590 } else { 6591 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) 6592 .addImm(SPOffset) 6593 .addReg(BufReg); 6594 } 6595 MIB.setMemRefs(MMOBegin, MMOEnd); 6596 6597 // Reload BP 6598 if (PVT == MVT::i64) { 6599 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) 6600 .addImm(BPOffset) 6601 .addReg(BufReg); 6602 } else { 6603 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) 6604 .addImm(BPOffset) 6605 .addReg(BufReg); 6606 } 6607 MIB.setMemRefs(MMOBegin, MMOEnd); 6608 6609 // Reload TOC 6610 if (PVT == MVT::i64 && PPCSubTarget.isSVR4ABI()) { 6611 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) 6612 .addImm(TOCOffset) 6613 .addReg(BufReg); 6614 6615 MIB.setMemRefs(MMOBegin, MMOEnd); 6616 } 6617 6618 // Jump 6619 BuildMI(*MBB, MI, DL, 6620 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); 6621 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); 6622 6623 MI->eraseFromParent(); 6624 return MBB; 6625} 6626 6627MachineBasicBlock * 6628PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 6629 MachineBasicBlock *BB) const { 6630 if (MI->getOpcode() == PPC::EH_SjLj_SetJmp32 || 6631 MI->getOpcode() == PPC::EH_SjLj_SetJmp64) { 6632 return emitEHSjLjSetJmp(MI, BB); 6633 } else if (MI->getOpcode() == PPC::EH_SjLj_LongJmp32 || 6634 MI->getOpcode() == PPC::EH_SjLj_LongJmp64) { 6635 return emitEHSjLjLongJmp(MI, BB); 6636 } 6637 6638 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6639 6640 // To "insert" these instructions we actually have to insert their 6641 // control-flow patterns. 6642 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 6643 MachineFunction::iterator It = BB; 6644 ++It; 6645 6646 MachineFunction *F = BB->getParent(); 6647 6648 if (PPCSubTarget.hasISEL() && (MI->getOpcode() == PPC::SELECT_CC_I4 || 6649 MI->getOpcode() == PPC::SELECT_CC_I8 || 6650 MI->getOpcode() == PPC::SELECT_I4 || 6651 MI->getOpcode() == PPC::SELECT_I8)) { 6652 SmallVector<MachineOperand, 2> Cond; 6653 if (MI->getOpcode() == PPC::SELECT_CC_I4 || 6654 MI->getOpcode() == PPC::SELECT_CC_I8) 6655 Cond.push_back(MI->getOperand(4)); 6656 else 6657 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 6658 Cond.push_back(MI->getOperand(1)); 6659 6660 DebugLoc dl = MI->getDebugLoc(); 6661 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 6662 TII->insertSelect(*BB, MI, dl, MI->getOperand(0).getReg(), 6663 Cond, MI->getOperand(2).getReg(), 6664 MI->getOperand(3).getReg()); 6665 } else if (MI->getOpcode() == PPC::SELECT_CC_I4 || 6666 MI->getOpcode() == PPC::SELECT_CC_I8 || 6667 MI->getOpcode() == PPC::SELECT_CC_F4 || 6668 MI->getOpcode() == PPC::SELECT_CC_F8 || 6669 MI->getOpcode() == PPC::SELECT_CC_VRRC || 6670 MI->getOpcode() == PPC::SELECT_I4 || 6671 MI->getOpcode() == PPC::SELECT_I8 || 6672 MI->getOpcode() == PPC::SELECT_F4 || 6673 MI->getOpcode() == PPC::SELECT_F8 || 6674 MI->getOpcode() == PPC::SELECT_VRRC) { 6675 // The incoming instruction knows the destination vreg to set, the 6676 // condition code register to branch on, the true/false values to 6677 // select between, and a branch opcode to use. 6678 6679 // thisMBB: 6680 // ... 6681 // TrueVal = ... 6682 // cmpTY ccX, r1, r2 6683 // bCC copy1MBB 6684 // fallthrough --> copy0MBB 6685 MachineBasicBlock *thisMBB = BB; 6686 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 6687 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 6688 DebugLoc dl = MI->getDebugLoc(); 6689 F->insert(It, copy0MBB); 6690 F->insert(It, sinkMBB); 6691 6692 // Transfer the remainder of BB and its successor edges to sinkMBB. 6693 sinkMBB->splice(sinkMBB->begin(), BB, 6694 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 6695 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 6696 6697 // Next, add the true and fallthrough blocks as its successors. 6698 BB->addSuccessor(copy0MBB); 6699 BB->addSuccessor(sinkMBB); 6700 6701 if (MI->getOpcode() == PPC::SELECT_I4 || 6702 MI->getOpcode() == PPC::SELECT_I8 || 6703 MI->getOpcode() == PPC::SELECT_F4 || 6704 MI->getOpcode() == PPC::SELECT_F8 || 6705 MI->getOpcode() == PPC::SELECT_VRRC) { 6706 BuildMI(BB, dl, TII->get(PPC::BC)) 6707 .addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB); 6708 } else { 6709 unsigned SelectPred = MI->getOperand(4).getImm(); 6710 BuildMI(BB, dl, TII->get(PPC::BCC)) 6711 .addImm(SelectPred).addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB); 6712 } 6713 6714 // copy0MBB: 6715 // %FalseValue = ... 6716 // # fallthrough to sinkMBB 6717 BB = copy0MBB; 6718 6719 // Update machine-CFG edges 6720 BB->addSuccessor(sinkMBB); 6721 6722 // sinkMBB: 6723 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 6724 // ... 6725 BB = sinkMBB; 6726 BuildMI(*BB, BB->begin(), dl, 6727 TII->get(PPC::PHI), MI->getOperand(0).getReg()) 6728 .addReg(MI->getOperand(3).getReg()).addMBB(copy0MBB) 6729 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 6730 } 6731 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) 6732 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); 6733 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) 6734 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); 6735 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) 6736 BB = EmitAtomicBinary(MI, BB, false, PPC::ADD4); 6737 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) 6738 BB = EmitAtomicBinary(MI, BB, true, PPC::ADD8); 6739 6740 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I8) 6741 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); 6742 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I16) 6743 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); 6744 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I32) 6745 BB = EmitAtomicBinary(MI, BB, false, PPC::AND); 6746 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I64) 6747 BB = EmitAtomicBinary(MI, BB, true, PPC::AND8); 6748 6749 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I8) 6750 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); 6751 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I16) 6752 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); 6753 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I32) 6754 BB = EmitAtomicBinary(MI, BB, false, PPC::OR); 6755 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I64) 6756 BB = EmitAtomicBinary(MI, BB, true, PPC::OR8); 6757 6758 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) 6759 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); 6760 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) 6761 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); 6762 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) 6763 BB = EmitAtomicBinary(MI, BB, false, PPC::XOR); 6764 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) 6765 BB = EmitAtomicBinary(MI, BB, true, PPC::XOR8); 6766 6767 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) 6768 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ANDC); 6769 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) 6770 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ANDC); 6771 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) 6772 BB = EmitAtomicBinary(MI, BB, false, PPC::ANDC); 6773 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) 6774 BB = EmitAtomicBinary(MI, BB, true, PPC::ANDC8); 6775 6776 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) 6777 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); 6778 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) 6779 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); 6780 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) 6781 BB = EmitAtomicBinary(MI, BB, false, PPC::SUBF); 6782 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) 6783 BB = EmitAtomicBinary(MI, BB, true, PPC::SUBF8); 6784 6785 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I8) 6786 BB = EmitPartwordAtomicBinary(MI, BB, true, 0); 6787 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I16) 6788 BB = EmitPartwordAtomicBinary(MI, BB, false, 0); 6789 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I32) 6790 BB = EmitAtomicBinary(MI, BB, false, 0); 6791 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I64) 6792 BB = EmitAtomicBinary(MI, BB, true, 0); 6793 6794 else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || 6795 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64) { 6796 bool is64bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; 6797 6798 unsigned dest = MI->getOperand(0).getReg(); 6799 unsigned ptrA = MI->getOperand(1).getReg(); 6800 unsigned ptrB = MI->getOperand(2).getReg(); 6801 unsigned oldval = MI->getOperand(3).getReg(); 6802 unsigned newval = MI->getOperand(4).getReg(); 6803 DebugLoc dl = MI->getDebugLoc(); 6804 6805 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 6806 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 6807 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 6808 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 6809 F->insert(It, loop1MBB); 6810 F->insert(It, loop2MBB); 6811 F->insert(It, midMBB); 6812 F->insert(It, exitMBB); 6813 exitMBB->splice(exitMBB->begin(), BB, 6814 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 6815 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 6816 6817 // thisMBB: 6818 // ... 6819 // fallthrough --> loopMBB 6820 BB->addSuccessor(loop1MBB); 6821 6822 // loop1MBB: 6823 // l[wd]arx dest, ptr 6824 // cmp[wd] dest, oldval 6825 // bne- midMBB 6826 // loop2MBB: 6827 // st[wd]cx. newval, ptr 6828 // bne- loopMBB 6829 // b exitBB 6830 // midMBB: 6831 // st[wd]cx. dest, ptr 6832 // exitBB: 6833 BB = loop1MBB; 6834 BuildMI(BB, dl, TII->get(is64bit ? PPC::LDARX : PPC::LWARX), dest) 6835 .addReg(ptrA).addReg(ptrB); 6836 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) 6837 .addReg(oldval).addReg(dest); 6838 BuildMI(BB, dl, TII->get(PPC::BCC)) 6839 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB); 6840 BB->addSuccessor(loop2MBB); 6841 BB->addSuccessor(midMBB); 6842 6843 BB = loop2MBB; 6844 BuildMI(BB, dl, TII->get(is64bit ? PPC::STDCX : PPC::STWCX)) 6845 .addReg(newval).addReg(ptrA).addReg(ptrB); 6846 BuildMI(BB, dl, TII->get(PPC::BCC)) 6847 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB); 6848 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 6849 BB->addSuccessor(loop1MBB); 6850 BB->addSuccessor(exitMBB); 6851 6852 BB = midMBB; 6853 BuildMI(BB, dl, TII->get(is64bit ? PPC::STDCX : PPC::STWCX)) 6854 .addReg(dest).addReg(ptrA).addReg(ptrB); 6855 BB->addSuccessor(exitMBB); 6856 6857 // exitMBB: 6858 // ... 6859 BB = exitMBB; 6860 } else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || 6861 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { 6862 // We must use 64-bit registers for addresses when targeting 64-bit, 6863 // since we're actually doing arithmetic on them. Other registers 6864 // can be 32-bit. 6865 bool is64bit = PPCSubTarget.isPPC64(); 6866 bool is8bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; 6867 6868 unsigned dest = MI->getOperand(0).getReg(); 6869 unsigned ptrA = MI->getOperand(1).getReg(); 6870 unsigned ptrB = MI->getOperand(2).getReg(); 6871 unsigned oldval = MI->getOperand(3).getReg(); 6872 unsigned newval = MI->getOperand(4).getReg(); 6873 DebugLoc dl = MI->getDebugLoc(); 6874 6875 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 6876 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 6877 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 6878 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 6879 F->insert(It, loop1MBB); 6880 F->insert(It, loop2MBB); 6881 F->insert(It, midMBB); 6882 F->insert(It, exitMBB); 6883 exitMBB->splice(exitMBB->begin(), BB, 6884 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 6885 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 6886 6887 MachineRegisterInfo &RegInfo = F->getRegInfo(); 6888 const TargetRegisterClass *RC = 6889 is64bit ? (const TargetRegisterClass *) &PPC::G8RCRegClass : 6890 (const TargetRegisterClass *) &PPC::GPRCRegClass; 6891 unsigned PtrReg = RegInfo.createVirtualRegister(RC); 6892 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC); 6893 unsigned ShiftReg = RegInfo.createVirtualRegister(RC); 6894 unsigned NewVal2Reg = RegInfo.createVirtualRegister(RC); 6895 unsigned NewVal3Reg = RegInfo.createVirtualRegister(RC); 6896 unsigned OldVal2Reg = RegInfo.createVirtualRegister(RC); 6897 unsigned OldVal3Reg = RegInfo.createVirtualRegister(RC); 6898 unsigned MaskReg = RegInfo.createVirtualRegister(RC); 6899 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC); 6900 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC); 6901 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC); 6902 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC); 6903 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC); 6904 unsigned Ptr1Reg; 6905 unsigned TmpReg = RegInfo.createVirtualRegister(RC); 6906 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 6907 // thisMBB: 6908 // ... 6909 // fallthrough --> loopMBB 6910 BB->addSuccessor(loop1MBB); 6911 6912 // The 4-byte load must be aligned, while a char or short may be 6913 // anywhere in the word. Hence all this nasty bookkeeping code. 6914 // add ptr1, ptrA, ptrB [copy if ptrA==0] 6915 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 6916 // xori shift, shift1, 24 [16] 6917 // rlwinm ptr, ptr1, 0, 0, 29 6918 // slw newval2, newval, shift 6919 // slw oldval2, oldval,shift 6920 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 6921 // slw mask, mask2, shift 6922 // and newval3, newval2, mask 6923 // and oldval3, oldval2, mask 6924 // loop1MBB: 6925 // lwarx tmpDest, ptr 6926 // and tmp, tmpDest, mask 6927 // cmpw tmp, oldval3 6928 // bne- midMBB 6929 // loop2MBB: 6930 // andc tmp2, tmpDest, mask 6931 // or tmp4, tmp2, newval3 6932 // stwcx. tmp4, ptr 6933 // bne- loop1MBB 6934 // b exitBB 6935 // midMBB: 6936 // stwcx. tmpDest, ptr 6937 // exitBB: 6938 // srw dest, tmpDest, shift 6939 if (ptrA != ZeroReg) { 6940 Ptr1Reg = RegInfo.createVirtualRegister(RC); 6941 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 6942 .addReg(ptrA).addReg(ptrB); 6943 } else { 6944 Ptr1Reg = ptrB; 6945 } 6946 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg) 6947 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27); 6948 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg) 6949 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16); 6950 if (is64bit) 6951 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 6952 .addReg(Ptr1Reg).addImm(0).addImm(61); 6953 else 6954 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 6955 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29); 6956 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) 6957 .addReg(newval).addReg(ShiftReg); 6958 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) 6959 .addReg(oldval).addReg(ShiftReg); 6960 if (is8bit) 6961 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 6962 else { 6963 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 6964 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 6965 .addReg(Mask3Reg).addImm(65535); 6966 } 6967 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 6968 .addReg(Mask2Reg).addReg(ShiftReg); 6969 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) 6970 .addReg(NewVal2Reg).addReg(MaskReg); 6971 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) 6972 .addReg(OldVal2Reg).addReg(MaskReg); 6973 6974 BB = loop1MBB; 6975 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 6976 .addReg(ZeroReg).addReg(PtrReg); 6977 BuildMI(BB, dl, TII->get(PPC::AND),TmpReg) 6978 .addReg(TmpDestReg).addReg(MaskReg); 6979 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) 6980 .addReg(TmpReg).addReg(OldVal3Reg); 6981 BuildMI(BB, dl, TII->get(PPC::BCC)) 6982 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB); 6983 BB->addSuccessor(loop2MBB); 6984 BB->addSuccessor(midMBB); 6985 6986 BB = loop2MBB; 6987 BuildMI(BB, dl, TII->get(PPC::ANDC),Tmp2Reg) 6988 .addReg(TmpDestReg).addReg(MaskReg); 6989 BuildMI(BB, dl, TII->get(PPC::OR),Tmp4Reg) 6990 .addReg(Tmp2Reg).addReg(NewVal3Reg); 6991 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(Tmp4Reg) 6992 .addReg(ZeroReg).addReg(PtrReg); 6993 BuildMI(BB, dl, TII->get(PPC::BCC)) 6994 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB); 6995 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 6996 BB->addSuccessor(loop1MBB); 6997 BB->addSuccessor(exitMBB); 6998 6999 BB = midMBB; 7000 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(TmpDestReg) 7001 .addReg(ZeroReg).addReg(PtrReg); 7002 BB->addSuccessor(exitMBB); 7003 7004 // exitMBB: 7005 // ... 7006 BB = exitMBB; 7007 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW),dest).addReg(TmpReg) 7008 .addReg(ShiftReg); 7009 } else if (MI->getOpcode() == PPC::FADDrtz) { 7010 // This pseudo performs an FADD with rounding mode temporarily forced 7011 // to round-to-zero. We emit this via custom inserter since the FPSCR 7012 // is not modeled at the SelectionDAG level. 7013 unsigned Dest = MI->getOperand(0).getReg(); 7014 unsigned Src1 = MI->getOperand(1).getReg(); 7015 unsigned Src2 = MI->getOperand(2).getReg(); 7016 DebugLoc dl = MI->getDebugLoc(); 7017 7018 MachineRegisterInfo &RegInfo = F->getRegInfo(); 7019 unsigned MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 7020 7021 // Save FPSCR value. 7022 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); 7023 7024 // Set rounding mode to round-to-zero. 7025 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); 7026 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); 7027 7028 // Perform addition. 7029 BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); 7030 7031 // Restore FPSCR value. 7032 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)).addImm(1).addReg(MFFSReg); 7033 } else if (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT || 7034 MI->getOpcode() == PPC::ANDIo_1_GT_BIT || 7035 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 7036 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) { 7037 unsigned Opcode = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 7038 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) ? 7039 PPC::ANDIo8 : PPC::ANDIo; 7040 bool isEQ = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT || 7041 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8); 7042 7043 MachineRegisterInfo &RegInfo = F->getRegInfo(); 7044 unsigned Dest = RegInfo.createVirtualRegister(Opcode == PPC::ANDIo ? 7045 &PPC::GPRCRegClass : 7046 &PPC::G8RCRegClass); 7047 7048 DebugLoc dl = MI->getDebugLoc(); 7049 BuildMI(*BB, MI, dl, TII->get(Opcode), Dest) 7050 .addReg(MI->getOperand(1).getReg()).addImm(1); 7051 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), 7052 MI->getOperand(0).getReg()) 7053 .addReg(isEQ ? PPC::CR0EQ : PPC::CR0GT); 7054 } else { 7055 llvm_unreachable("Unexpected instr type to insert"); 7056 } 7057 7058 MI->eraseFromParent(); // The pseudo instruction is gone now. 7059 return BB; 7060} 7061 7062//===----------------------------------------------------------------------===// 7063// Target Optimization Hooks 7064//===----------------------------------------------------------------------===// 7065 7066SDValue PPCTargetLowering::DAGCombineFastRecip(SDValue Op, 7067 DAGCombinerInfo &DCI) const { 7068 if (DCI.isAfterLegalizeVectorOps()) 7069 return SDValue(); 7070 7071 EVT VT = Op.getValueType(); 7072 7073 if ((VT == MVT::f32 && PPCSubTarget.hasFRES()) || 7074 (VT == MVT::f64 && PPCSubTarget.hasFRE()) || 7075 (VT == MVT::v4f32 && PPCSubTarget.hasAltivec()) || 7076 (VT == MVT::v2f64 && PPCSubTarget.hasVSX())) { 7077 7078 // Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i) 7079 // For the reciprocal, we need to find the zero of the function: 7080 // F(X) = A X - 1 [which has a zero at X = 1/A] 7081 // => 7082 // X_{i+1} = X_i (2 - A X_i) = X_i + X_i (1 - A X_i) [this second form 7083 // does not require additional intermediate precision] 7084 7085 // Convergence is quadratic, so we essentially double the number of digits 7086 // correct after every iteration. The minimum architected relative 7087 // accuracy is 2^-5. When hasRecipPrec(), this is 2^-14. IEEE float has 7088 // 23 digits and double has 52 digits. 7089 int Iterations = PPCSubTarget.hasRecipPrec() ? 1 : 3; 7090 if (VT.getScalarType() == MVT::f64) 7091 ++Iterations; 7092 7093 SelectionDAG &DAG = DCI.DAG; 7094 SDLoc dl(Op); 7095 7096 SDValue FPOne = 7097 DAG.getConstantFP(1.0, VT.getScalarType()); 7098 if (VT.isVector()) { 7099 assert(VT.getVectorNumElements() == 4 && 7100 "Unknown vector type"); 7101 FPOne = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, 7102 FPOne, FPOne, FPOne, FPOne); 7103 } 7104 7105 SDValue Est = DAG.getNode(PPCISD::FRE, dl, VT, Op); 7106 DCI.AddToWorklist(Est.getNode()); 7107 7108 // Newton iterations: Est = Est + Est (1 - Arg * Est) 7109 for (int i = 0; i < Iterations; ++i) { 7110 SDValue NewEst = DAG.getNode(ISD::FMUL, dl, VT, Op, Est); 7111 DCI.AddToWorklist(NewEst.getNode()); 7112 7113 NewEst = DAG.getNode(ISD::FSUB, dl, VT, FPOne, NewEst); 7114 DCI.AddToWorklist(NewEst.getNode()); 7115 7116 NewEst = DAG.getNode(ISD::FMUL, dl, VT, Est, NewEst); 7117 DCI.AddToWorklist(NewEst.getNode()); 7118 7119 Est = DAG.getNode(ISD::FADD, dl, VT, Est, NewEst); 7120 DCI.AddToWorklist(Est.getNode()); 7121 } 7122 7123 return Est; 7124 } 7125 7126 return SDValue(); 7127} 7128 7129SDValue PPCTargetLowering::DAGCombineFastRecipFSQRT(SDValue Op, 7130 DAGCombinerInfo &DCI) const { 7131 if (DCI.isAfterLegalizeVectorOps()) 7132 return SDValue(); 7133 7134 EVT VT = Op.getValueType(); 7135 7136 if ((VT == MVT::f32 && PPCSubTarget.hasFRSQRTES()) || 7137 (VT == MVT::f64 && PPCSubTarget.hasFRSQRTE()) || 7138 (VT == MVT::v4f32 && PPCSubTarget.hasAltivec()) || 7139 (VT == MVT::v2f64 && PPCSubTarget.hasVSX())) { 7140 7141 // Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i) 7142 // For the reciprocal sqrt, we need to find the zero of the function: 7143 // F(X) = 1/X^2 - A [which has a zero at X = 1/sqrt(A)] 7144 // => 7145 // X_{i+1} = X_i (1.5 - A X_i^2 / 2) 7146 // As a result, we precompute A/2 prior to the iteration loop. 7147 7148 // Convergence is quadratic, so we essentially double the number of digits 7149 // correct after every iteration. The minimum architected relative 7150 // accuracy is 2^-5. When hasRecipPrec(), this is 2^-14. IEEE float has 7151 // 23 digits and double has 52 digits. 7152 int Iterations = PPCSubTarget.hasRecipPrec() ? 1 : 3; 7153 if (VT.getScalarType() == MVT::f64) 7154 ++Iterations; 7155 7156 SelectionDAG &DAG = DCI.DAG; 7157 SDLoc dl(Op); 7158 7159 SDValue FPThreeHalves = 7160 DAG.getConstantFP(1.5, VT.getScalarType()); 7161 if (VT.isVector()) { 7162 assert(VT.getVectorNumElements() == 4 && 7163 "Unknown vector type"); 7164 FPThreeHalves = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, 7165 FPThreeHalves, FPThreeHalves, 7166 FPThreeHalves, FPThreeHalves); 7167 } 7168 7169 SDValue Est = DAG.getNode(PPCISD::FRSQRTE, dl, VT, Op); 7170 DCI.AddToWorklist(Est.getNode()); 7171 7172 // We now need 0.5*Arg which we can write as (1.5*Arg - Arg) so that 7173 // this entire sequence requires only one FP constant. 7174 SDValue HalfArg = DAG.getNode(ISD::FMUL, dl, VT, FPThreeHalves, Op); 7175 DCI.AddToWorklist(HalfArg.getNode()); 7176 7177 HalfArg = DAG.getNode(ISD::FSUB, dl, VT, HalfArg, Op); 7178 DCI.AddToWorklist(HalfArg.getNode()); 7179 7180 // Newton iterations: Est = Est * (1.5 - HalfArg * Est * Est) 7181 for (int i = 0; i < Iterations; ++i) { 7182 SDValue NewEst = DAG.getNode(ISD::FMUL, dl, VT, Est, Est); 7183 DCI.AddToWorklist(NewEst.getNode()); 7184 7185 NewEst = DAG.getNode(ISD::FMUL, dl, VT, HalfArg, NewEst); 7186 DCI.AddToWorklist(NewEst.getNode()); 7187 7188 NewEst = DAG.getNode(ISD::FSUB, dl, VT, FPThreeHalves, NewEst); 7189 DCI.AddToWorklist(NewEst.getNode()); 7190 7191 Est = DAG.getNode(ISD::FMUL, dl, VT, Est, NewEst); 7192 DCI.AddToWorklist(Est.getNode()); 7193 } 7194 7195 return Est; 7196 } 7197 7198 return SDValue(); 7199} 7200 7201// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does 7202// not enforce equality of the chain operands. 7203static bool isConsecutiveLS(LSBaseSDNode *LS, LSBaseSDNode *Base, 7204 unsigned Bytes, int Dist, 7205 SelectionDAG &DAG) { 7206 EVT VT = LS->getMemoryVT(); 7207 if (VT.getSizeInBits() / 8 != Bytes) 7208 return false; 7209 7210 SDValue Loc = LS->getBasePtr(); 7211 SDValue BaseLoc = Base->getBasePtr(); 7212 if (Loc.getOpcode() == ISD::FrameIndex) { 7213 if (BaseLoc.getOpcode() != ISD::FrameIndex) 7214 return false; 7215 const MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 7216 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 7217 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 7218 int FS = MFI->getObjectSize(FI); 7219 int BFS = MFI->getObjectSize(BFI); 7220 if (FS != BFS || FS != (int)Bytes) return false; 7221 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 7222 } 7223 7224 // Handle X+C 7225 if (DAG.isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 7226 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 7227 return true; 7228 7229 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7230 const GlobalValue *GV1 = nullptr; 7231 const GlobalValue *GV2 = nullptr; 7232 int64_t Offset1 = 0; 7233 int64_t Offset2 = 0; 7234 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 7235 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 7236 if (isGA1 && isGA2 && GV1 == GV2) 7237 return Offset1 == (Offset2 + Dist*Bytes); 7238 return false; 7239} 7240 7241// Return true is there is a nearyby consecutive load to the one provided 7242// (regardless of alignment). We search up and down the chain, looking though 7243// token factors and other loads (but nothing else). As a result, a true 7244// results indicates that it is safe to create a new consecutive load adjacent 7245// to the load provided. 7246static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { 7247 SDValue Chain = LD->getChain(); 7248 EVT VT = LD->getMemoryVT(); 7249 7250 SmallSet<SDNode *, 16> LoadRoots; 7251 SmallVector<SDNode *, 8> Queue(1, Chain.getNode()); 7252 SmallSet<SDNode *, 16> Visited; 7253 7254 // First, search up the chain, branching to follow all token-factor operands. 7255 // If we find a consecutive load, then we're done, otherwise, record all 7256 // nodes just above the top-level loads and token factors. 7257 while (!Queue.empty()) { 7258 SDNode *ChainNext = Queue.pop_back_val(); 7259 if (!Visited.insert(ChainNext)) 7260 continue; 7261 7262 if (LoadSDNode *ChainLD = dyn_cast<LoadSDNode>(ChainNext)) { 7263 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 7264 return true; 7265 7266 if (!Visited.count(ChainLD->getChain().getNode())) 7267 Queue.push_back(ChainLD->getChain().getNode()); 7268 } else if (ChainNext->getOpcode() == ISD::TokenFactor) { 7269 for (SDNode::op_iterator O = ChainNext->op_begin(), 7270 OE = ChainNext->op_end(); O != OE; ++O) 7271 if (!Visited.count(O->getNode())) 7272 Queue.push_back(O->getNode()); 7273 } else 7274 LoadRoots.insert(ChainNext); 7275 } 7276 7277 // Second, search down the chain, starting from the top-level nodes recorded 7278 // in the first phase. These top-level nodes are the nodes just above all 7279 // loads and token factors. Starting with their uses, recursively look though 7280 // all loads (just the chain uses) and token factors to find a consecutive 7281 // load. 7282 Visited.clear(); 7283 Queue.clear(); 7284 7285 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(), 7286 IE = LoadRoots.end(); I != IE; ++I) { 7287 Queue.push_back(*I); 7288 7289 while (!Queue.empty()) { 7290 SDNode *LoadRoot = Queue.pop_back_val(); 7291 if (!Visited.insert(LoadRoot)) 7292 continue; 7293 7294 if (LoadSDNode *ChainLD = dyn_cast<LoadSDNode>(LoadRoot)) 7295 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 7296 return true; 7297 7298 for (SDNode::use_iterator UI = LoadRoot->use_begin(), 7299 UE = LoadRoot->use_end(); UI != UE; ++UI) 7300 if (((isa<LoadSDNode>(*UI) && 7301 cast<LoadSDNode>(*UI)->getChain().getNode() == LoadRoot) || 7302 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) 7303 Queue.push_back(*UI); 7304 } 7305 } 7306 7307 return false; 7308} 7309 7310SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, 7311 DAGCombinerInfo &DCI) const { 7312 SelectionDAG &DAG = DCI.DAG; 7313 SDLoc dl(N); 7314 7315 assert(PPCSubTarget.useCRBits() && 7316 "Expecting to be tracking CR bits"); 7317 // If we're tracking CR bits, we need to be careful that we don't have: 7318 // trunc(binary-ops(zext(x), zext(y))) 7319 // or 7320 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) 7321 // such that we're unnecessarily moving things into GPRs when it would be 7322 // better to keep them in CR bits. 7323 7324 // Note that trunc here can be an actual i1 trunc, or can be the effective 7325 // truncation that comes from a setcc or select_cc. 7326 if (N->getOpcode() == ISD::TRUNCATE && 7327 N->getValueType(0) != MVT::i1) 7328 return SDValue(); 7329 7330 if (N->getOperand(0).getValueType() != MVT::i32 && 7331 N->getOperand(0).getValueType() != MVT::i64) 7332 return SDValue(); 7333 7334 if (N->getOpcode() == ISD::SETCC || 7335 N->getOpcode() == ISD::SELECT_CC) { 7336 // If we're looking at a comparison, then we need to make sure that the 7337 // high bits (all except for the first) don't matter the result. 7338 ISD::CondCode CC = 7339 cast<CondCodeSDNode>(N->getOperand( 7340 N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); 7341 unsigned OpBits = N->getOperand(0).getValueSizeInBits(); 7342 7343 if (ISD::isSignedIntSetCC(CC)) { 7344 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || 7345 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) 7346 return SDValue(); 7347 } else if (ISD::isUnsignedIntSetCC(CC)) { 7348 if (!DAG.MaskedValueIsZero(N->getOperand(0), 7349 APInt::getHighBitsSet(OpBits, OpBits-1)) || 7350 !DAG.MaskedValueIsZero(N->getOperand(1), 7351 APInt::getHighBitsSet(OpBits, OpBits-1))) 7352 return SDValue(); 7353 } else { 7354 // This is neither a signed nor an unsigned comparison, just make sure 7355 // that the high bits are equal. 7356 APInt Op1Zero, Op1One; 7357 APInt Op2Zero, Op2One; 7358 DAG.computeKnownBits(N->getOperand(0), Op1Zero, Op1One); 7359 DAG.computeKnownBits(N->getOperand(1), Op2Zero, Op2One); 7360 7361 // We don't really care about what is known about the first bit (if 7362 // anything), so clear it in all masks prior to comparing them. 7363 Op1Zero.clearBit(0); Op1One.clearBit(0); 7364 Op2Zero.clearBit(0); Op2One.clearBit(0); 7365 7366 if (Op1Zero != Op2Zero || Op1One != Op2One) 7367 return SDValue(); 7368 } 7369 } 7370 7371 // We now know that the higher-order bits are irrelevant, we just need to 7372 // make sure that all of the intermediate operations are bit operations, and 7373 // all inputs are extensions. 7374 if (N->getOperand(0).getOpcode() != ISD::AND && 7375 N->getOperand(0).getOpcode() != ISD::OR && 7376 N->getOperand(0).getOpcode() != ISD::XOR && 7377 N->getOperand(0).getOpcode() != ISD::SELECT && 7378 N->getOperand(0).getOpcode() != ISD::SELECT_CC && 7379 N->getOperand(0).getOpcode() != ISD::TRUNCATE && 7380 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && 7381 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && 7382 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) 7383 return SDValue(); 7384 7385 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && 7386 N->getOperand(1).getOpcode() != ISD::AND && 7387 N->getOperand(1).getOpcode() != ISD::OR && 7388 N->getOperand(1).getOpcode() != ISD::XOR && 7389 N->getOperand(1).getOpcode() != ISD::SELECT && 7390 N->getOperand(1).getOpcode() != ISD::SELECT_CC && 7391 N->getOperand(1).getOpcode() != ISD::TRUNCATE && 7392 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && 7393 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && 7394 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) 7395 return SDValue(); 7396 7397 SmallVector<SDValue, 4> Inputs; 7398 SmallVector<SDValue, 8> BinOps, PromOps; 7399 SmallPtrSet<SDNode *, 16> Visited; 7400 7401 for (unsigned i = 0; i < 2; ++i) { 7402 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 7403 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 7404 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 7405 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || 7406 isa<ConstantSDNode>(N->getOperand(i))) 7407 Inputs.push_back(N->getOperand(i)); 7408 else 7409 BinOps.push_back(N->getOperand(i)); 7410 7411 if (N->getOpcode() == ISD::TRUNCATE) 7412 break; 7413 } 7414 7415 // Visit all inputs, collect all binary operations (and, or, xor and 7416 // select) that are all fed by extensions. 7417 while (!BinOps.empty()) { 7418 SDValue BinOp = BinOps.back(); 7419 BinOps.pop_back(); 7420 7421 if (!Visited.insert(BinOp.getNode())) 7422 continue; 7423 7424 PromOps.push_back(BinOp); 7425 7426 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 7427 // The condition of the select is not promoted. 7428 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 7429 continue; 7430 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 7431 continue; 7432 7433 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 7434 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 7435 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 7436 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || 7437 isa<ConstantSDNode>(BinOp.getOperand(i))) { 7438 Inputs.push_back(BinOp.getOperand(i)); 7439 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 7440 BinOp.getOperand(i).getOpcode() == ISD::OR || 7441 BinOp.getOperand(i).getOpcode() == ISD::XOR || 7442 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 7443 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || 7444 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 7445 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 7446 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 7447 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { 7448 BinOps.push_back(BinOp.getOperand(i)); 7449 } else { 7450 // We have an input that is not an extension or another binary 7451 // operation; we'll abort this transformation. 7452 return SDValue(); 7453 } 7454 } 7455 } 7456 7457 // Make sure that this is a self-contained cluster of operations (which 7458 // is not quite the same thing as saying that everything has only one 7459 // use). 7460 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 7461 if (isa<ConstantSDNode>(Inputs[i])) 7462 continue; 7463 7464 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 7465 UE = Inputs[i].getNode()->use_end(); 7466 UI != UE; ++UI) { 7467 SDNode *User = *UI; 7468 if (User != N && !Visited.count(User)) 7469 return SDValue(); 7470 7471 // Make sure that we're not going to promote the non-output-value 7472 // operand(s) or SELECT or SELECT_CC. 7473 // FIXME: Although we could sometimes handle this, and it does occur in 7474 // practice that one of the condition inputs to the select is also one of 7475 // the outputs, we currently can't deal with this. 7476 if (User->getOpcode() == ISD::SELECT) { 7477 if (User->getOperand(0) == Inputs[i]) 7478 return SDValue(); 7479 } else if (User->getOpcode() == ISD::SELECT_CC) { 7480 if (User->getOperand(0) == Inputs[i] || 7481 User->getOperand(1) == Inputs[i]) 7482 return SDValue(); 7483 } 7484 } 7485 } 7486 7487 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 7488 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 7489 UE = PromOps[i].getNode()->use_end(); 7490 UI != UE; ++UI) { 7491 SDNode *User = *UI; 7492 if (User != N && !Visited.count(User)) 7493 return SDValue(); 7494 7495 // Make sure that we're not going to promote the non-output-value 7496 // operand(s) or SELECT or SELECT_CC. 7497 // FIXME: Although we could sometimes handle this, and it does occur in 7498 // practice that one of the condition inputs to the select is also one of 7499 // the outputs, we currently can't deal with this. 7500 if (User->getOpcode() == ISD::SELECT) { 7501 if (User->getOperand(0) == PromOps[i]) 7502 return SDValue(); 7503 } else if (User->getOpcode() == ISD::SELECT_CC) { 7504 if (User->getOperand(0) == PromOps[i] || 7505 User->getOperand(1) == PromOps[i]) 7506 return SDValue(); 7507 } 7508 } 7509 } 7510 7511 // Replace all inputs with the extension operand. 7512 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 7513 // Constants may have users outside the cluster of to-be-promoted nodes, 7514 // and so we need to replace those as we do the promotions. 7515 if (isa<ConstantSDNode>(Inputs[i])) 7516 continue; 7517 else 7518 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); 7519 } 7520 7521 // Replace all operations (these are all the same, but have a different 7522 // (i1) return type). DAG.getNode will validate that the types of 7523 // a binary operator match, so go through the list in reverse so that 7524 // we've likely promoted both operands first. Any intermediate truncations or 7525 // extensions disappear. 7526 while (!PromOps.empty()) { 7527 SDValue PromOp = PromOps.back(); 7528 PromOps.pop_back(); 7529 7530 if (PromOp.getOpcode() == ISD::TRUNCATE || 7531 PromOp.getOpcode() == ISD::SIGN_EXTEND || 7532 PromOp.getOpcode() == ISD::ZERO_EXTEND || 7533 PromOp.getOpcode() == ISD::ANY_EXTEND) { 7534 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) && 7535 PromOp.getOperand(0).getValueType() != MVT::i1) { 7536 // The operand is not yet ready (see comment below). 7537 PromOps.insert(PromOps.begin(), PromOp); 7538 continue; 7539 } 7540 7541 SDValue RepValue = PromOp.getOperand(0); 7542 if (isa<ConstantSDNode>(RepValue)) 7543 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); 7544 7545 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); 7546 continue; 7547 } 7548 7549 unsigned C; 7550 switch (PromOp.getOpcode()) { 7551 default: C = 0; break; 7552 case ISD::SELECT: C = 1; break; 7553 case ISD::SELECT_CC: C = 2; break; 7554 } 7555 7556 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 7557 PromOp.getOperand(C).getValueType() != MVT::i1) || 7558 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 7559 PromOp.getOperand(C+1).getValueType() != MVT::i1)) { 7560 // The to-be-promoted operands of this node have not yet been 7561 // promoted (this should be rare because we're going through the 7562 // list backward, but if one of the operands has several users in 7563 // this cluster of to-be-promoted nodes, it is possible). 7564 PromOps.insert(PromOps.begin(), PromOp); 7565 continue; 7566 } 7567 7568 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 7569 PromOp.getNode()->op_end()); 7570 7571 // If there are any constant inputs, make sure they're replaced now. 7572 for (unsigned i = 0; i < 2; ++i) 7573 if (isa<ConstantSDNode>(Ops[C+i])) 7574 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); 7575 7576 DAG.ReplaceAllUsesOfValueWith(PromOp, 7577 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); 7578 } 7579 7580 // Now we're left with the initial truncation itself. 7581 if (N->getOpcode() == ISD::TRUNCATE) 7582 return N->getOperand(0); 7583 7584 // Otherwise, this is a comparison. The operands to be compared have just 7585 // changed type (to i1), but everything else is the same. 7586 return SDValue(N, 0); 7587} 7588 7589SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, 7590 DAGCombinerInfo &DCI) const { 7591 SelectionDAG &DAG = DCI.DAG; 7592 SDLoc dl(N); 7593 7594 // If we're tracking CR bits, we need to be careful that we don't have: 7595 // zext(binary-ops(trunc(x), trunc(y))) 7596 // or 7597 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) 7598 // such that we're unnecessarily moving things into CR bits that can more 7599 // efficiently stay in GPRs. Note that if we're not certain that the high 7600 // bits are set as required by the final extension, we still may need to do 7601 // some masking to get the proper behavior. 7602 7603 // This same functionality is important on PPC64 when dealing with 7604 // 32-to-64-bit extensions; these occur often when 32-bit values are used as 7605 // the return values of functions. Because it is so similar, it is handled 7606 // here as well. 7607 7608 if (N->getValueType(0) != MVT::i32 && 7609 N->getValueType(0) != MVT::i64) 7610 return SDValue(); 7611 7612 if (!((N->getOperand(0).getValueType() == MVT::i1 && 7613 PPCSubTarget.useCRBits()) || 7614 (N->getOperand(0).getValueType() == MVT::i32 && 7615 PPCSubTarget.isPPC64()))) 7616 return SDValue(); 7617 7618 if (N->getOperand(0).getOpcode() != ISD::AND && 7619 N->getOperand(0).getOpcode() != ISD::OR && 7620 N->getOperand(0).getOpcode() != ISD::XOR && 7621 N->getOperand(0).getOpcode() != ISD::SELECT && 7622 N->getOperand(0).getOpcode() != ISD::SELECT_CC) 7623 return SDValue(); 7624 7625 SmallVector<SDValue, 4> Inputs; 7626 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps; 7627 SmallPtrSet<SDNode *, 16> Visited; 7628 7629 // Visit all inputs, collect all binary operations (and, or, xor and 7630 // select) that are all fed by truncations. 7631 while (!BinOps.empty()) { 7632 SDValue BinOp = BinOps.back(); 7633 BinOps.pop_back(); 7634 7635 if (!Visited.insert(BinOp.getNode())) 7636 continue; 7637 7638 PromOps.push_back(BinOp); 7639 7640 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 7641 // The condition of the select is not promoted. 7642 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 7643 continue; 7644 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 7645 continue; 7646 7647 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 7648 isa<ConstantSDNode>(BinOp.getOperand(i))) { 7649 Inputs.push_back(BinOp.getOperand(i)); 7650 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 7651 BinOp.getOperand(i).getOpcode() == ISD::OR || 7652 BinOp.getOperand(i).getOpcode() == ISD::XOR || 7653 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 7654 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { 7655 BinOps.push_back(BinOp.getOperand(i)); 7656 } else { 7657 // We have an input that is not a truncation or another binary 7658 // operation; we'll abort this transformation. 7659 return SDValue(); 7660 } 7661 } 7662 } 7663 7664 // Make sure that this is a self-contained cluster of operations (which 7665 // is not quite the same thing as saying that everything has only one 7666 // use). 7667 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 7668 if (isa<ConstantSDNode>(Inputs[i])) 7669 continue; 7670 7671 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 7672 UE = Inputs[i].getNode()->use_end(); 7673 UI != UE; ++UI) { 7674 SDNode *User = *UI; 7675 if (User != N && !Visited.count(User)) 7676 return SDValue(); 7677 7678 // Make sure that we're not going to promote the non-output-value 7679 // operand(s) or SELECT or SELECT_CC. 7680 // FIXME: Although we could sometimes handle this, and it does occur in 7681 // practice that one of the condition inputs to the select is also one of 7682 // the outputs, we currently can't deal with this. 7683 if (User->getOpcode() == ISD::SELECT) { 7684 if (User->getOperand(0) == Inputs[i]) 7685 return SDValue(); 7686 } else if (User->getOpcode() == ISD::SELECT_CC) { 7687 if (User->getOperand(0) == Inputs[i] || 7688 User->getOperand(1) == Inputs[i]) 7689 return SDValue(); 7690 } 7691 } 7692 } 7693 7694 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 7695 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 7696 UE = PromOps[i].getNode()->use_end(); 7697 UI != UE; ++UI) { 7698 SDNode *User = *UI; 7699 if (User != N && !Visited.count(User)) 7700 return SDValue(); 7701 7702 // Make sure that we're not going to promote the non-output-value 7703 // operand(s) or SELECT or SELECT_CC. 7704 // FIXME: Although we could sometimes handle this, and it does occur in 7705 // practice that one of the condition inputs to the select is also one of 7706 // the outputs, we currently can't deal with this. 7707 if (User->getOpcode() == ISD::SELECT) { 7708 if (User->getOperand(0) == PromOps[i]) 7709 return SDValue(); 7710 } else if (User->getOpcode() == ISD::SELECT_CC) { 7711 if (User->getOperand(0) == PromOps[i] || 7712 User->getOperand(1) == PromOps[i]) 7713 return SDValue(); 7714 } 7715 } 7716 } 7717 7718 unsigned PromBits = N->getOperand(0).getValueSizeInBits(); 7719 bool ReallyNeedsExt = false; 7720 if (N->getOpcode() != ISD::ANY_EXTEND) { 7721 // If all of the inputs are not already sign/zero extended, then 7722 // we'll still need to do that at the end. 7723 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 7724 if (isa<ConstantSDNode>(Inputs[i])) 7725 continue; 7726 7727 unsigned OpBits = 7728 Inputs[i].getOperand(0).getValueSizeInBits(); 7729 assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); 7730 7731 if ((N->getOpcode() == ISD::ZERO_EXTEND && 7732 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), 7733 APInt::getHighBitsSet(OpBits, 7734 OpBits-PromBits))) || 7735 (N->getOpcode() == ISD::SIGN_EXTEND && 7736 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < 7737 (OpBits-(PromBits-1)))) { 7738 ReallyNeedsExt = true; 7739 break; 7740 } 7741 } 7742 } 7743 7744 // Replace all inputs, either with the truncation operand, or a 7745 // truncation or extension to the final output type. 7746 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 7747 // Constant inputs need to be replaced with the to-be-promoted nodes that 7748 // use them because they might have users outside of the cluster of 7749 // promoted nodes. 7750 if (isa<ConstantSDNode>(Inputs[i])) 7751 continue; 7752 7753 SDValue InSrc = Inputs[i].getOperand(0); 7754 if (Inputs[i].getValueType() == N->getValueType(0)) 7755 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); 7756 else if (N->getOpcode() == ISD::SIGN_EXTEND) 7757 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 7758 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); 7759 else if (N->getOpcode() == ISD::ZERO_EXTEND) 7760 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 7761 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); 7762 else 7763 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 7764 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); 7765 } 7766 7767 // Replace all operations (these are all the same, but have a different 7768 // (promoted) return type). DAG.getNode will validate that the types of 7769 // a binary operator match, so go through the list in reverse so that 7770 // we've likely promoted both operands first. 7771 while (!PromOps.empty()) { 7772 SDValue PromOp = PromOps.back(); 7773 PromOps.pop_back(); 7774 7775 unsigned C; 7776 switch (PromOp.getOpcode()) { 7777 default: C = 0; break; 7778 case ISD::SELECT: C = 1; break; 7779 case ISD::SELECT_CC: C = 2; break; 7780 } 7781 7782 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 7783 PromOp.getOperand(C).getValueType() != N->getValueType(0)) || 7784 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 7785 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { 7786 // The to-be-promoted operands of this node have not yet been 7787 // promoted (this should be rare because we're going through the 7788 // list backward, but if one of the operands has several users in 7789 // this cluster of to-be-promoted nodes, it is possible). 7790 PromOps.insert(PromOps.begin(), PromOp); 7791 continue; 7792 } 7793 7794 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 7795 PromOp.getNode()->op_end()); 7796 7797 // If this node has constant inputs, then they'll need to be promoted here. 7798 for (unsigned i = 0; i < 2; ++i) { 7799 if (!isa<ConstantSDNode>(Ops[C+i])) 7800 continue; 7801 if (Ops[C+i].getValueType() == N->getValueType(0)) 7802 continue; 7803 7804 if (N->getOpcode() == ISD::SIGN_EXTEND) 7805 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 7806 else if (N->getOpcode() == ISD::ZERO_EXTEND) 7807 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 7808 else 7809 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 7810 } 7811 7812 DAG.ReplaceAllUsesOfValueWith(PromOp, 7813 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); 7814 } 7815 7816 // Now we're left with the initial extension itself. 7817 if (!ReallyNeedsExt) 7818 return N->getOperand(0); 7819 7820 // To zero extend, just mask off everything except for the first bit (in the 7821 // i1 case). 7822 if (N->getOpcode() == ISD::ZERO_EXTEND) 7823 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), 7824 DAG.getConstant(APInt::getLowBitsSet( 7825 N->getValueSizeInBits(0), PromBits), 7826 N->getValueType(0))); 7827 7828 assert(N->getOpcode() == ISD::SIGN_EXTEND && 7829 "Invalid extension type"); 7830 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0)); 7831 SDValue ShiftCst = 7832 DAG.getConstant(N->getValueSizeInBits(0)-PromBits, ShiftAmountTy); 7833 return DAG.getNode(ISD::SRA, dl, N->getValueType(0), 7834 DAG.getNode(ISD::SHL, dl, N->getValueType(0), 7835 N->getOperand(0), ShiftCst), ShiftCst); 7836} 7837 7838SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, 7839 DAGCombinerInfo &DCI) const { 7840 const TargetMachine &TM = getTargetMachine(); 7841 SelectionDAG &DAG = DCI.DAG; 7842 SDLoc dl(N); 7843 switch (N->getOpcode()) { 7844 default: break; 7845 case PPCISD::SHL: 7846 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 7847 if (C->isNullValue()) // 0 << V -> 0. 7848 return N->getOperand(0); 7849 } 7850 break; 7851 case PPCISD::SRL: 7852 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 7853 if (C->isNullValue()) // 0 >>u V -> 0. 7854 return N->getOperand(0); 7855 } 7856 break; 7857 case PPCISD::SRA: 7858 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 7859 if (C->isNullValue() || // 0 >>s V -> 0. 7860 C->isAllOnesValue()) // -1 >>s V -> -1. 7861 return N->getOperand(0); 7862 } 7863 break; 7864 case ISD::SIGN_EXTEND: 7865 case ISD::ZERO_EXTEND: 7866 case ISD::ANY_EXTEND: 7867 return DAGCombineExtBoolTrunc(N, DCI); 7868 case ISD::TRUNCATE: 7869 case ISD::SETCC: 7870 case ISD::SELECT_CC: 7871 return DAGCombineTruncBoolExt(N, DCI); 7872 case ISD::FDIV: { 7873 assert(TM.Options.UnsafeFPMath && 7874 "Reciprocal estimates require UnsafeFPMath"); 7875 7876 if (N->getOperand(1).getOpcode() == ISD::FSQRT) { 7877 SDValue RV = 7878 DAGCombineFastRecipFSQRT(N->getOperand(1).getOperand(0), DCI); 7879 if (RV.getNode()) { 7880 DCI.AddToWorklist(RV.getNode()); 7881 return DAG.getNode(ISD::FMUL, dl, N->getValueType(0), 7882 N->getOperand(0), RV); 7883 } 7884 } else if (N->getOperand(1).getOpcode() == ISD::FP_EXTEND && 7885 N->getOperand(1).getOperand(0).getOpcode() == ISD::FSQRT) { 7886 SDValue RV = 7887 DAGCombineFastRecipFSQRT(N->getOperand(1).getOperand(0).getOperand(0), 7888 DCI); 7889 if (RV.getNode()) { 7890 DCI.AddToWorklist(RV.getNode()); 7891 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N->getOperand(1)), 7892 N->getValueType(0), RV); 7893 DCI.AddToWorklist(RV.getNode()); 7894 return DAG.getNode(ISD::FMUL, dl, N->getValueType(0), 7895 N->getOperand(0), RV); 7896 } 7897 } else if (N->getOperand(1).getOpcode() == ISD::FP_ROUND && 7898 N->getOperand(1).getOperand(0).getOpcode() == ISD::FSQRT) { 7899 SDValue RV = 7900 DAGCombineFastRecipFSQRT(N->getOperand(1).getOperand(0).getOperand(0), 7901 DCI); 7902 if (RV.getNode()) { 7903 DCI.AddToWorklist(RV.getNode()); 7904 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N->getOperand(1)), 7905 N->getValueType(0), RV, 7906 N->getOperand(1).getOperand(1)); 7907 DCI.AddToWorklist(RV.getNode()); 7908 return DAG.getNode(ISD::FMUL, dl, N->getValueType(0), 7909 N->getOperand(0), RV); 7910 } 7911 } 7912 7913 SDValue RV = DAGCombineFastRecip(N->getOperand(1), DCI); 7914 if (RV.getNode()) { 7915 DCI.AddToWorklist(RV.getNode()); 7916 return DAG.getNode(ISD::FMUL, dl, N->getValueType(0), 7917 N->getOperand(0), RV); 7918 } 7919 7920 } 7921 break; 7922 case ISD::FSQRT: { 7923 assert(TM.Options.UnsafeFPMath && 7924 "Reciprocal estimates require UnsafeFPMath"); 7925 7926 // Compute this as 1/(1/sqrt(X)), which is the reciprocal of the 7927 // reciprocal sqrt. 7928 SDValue RV = DAGCombineFastRecipFSQRT(N->getOperand(0), DCI); 7929 if (RV.getNode()) { 7930 DCI.AddToWorklist(RV.getNode()); 7931 RV = DAGCombineFastRecip(RV, DCI); 7932 if (RV.getNode()) { 7933 // Unfortunately, RV is now NaN if the input was exactly 0. Select out 7934 // this case and force the answer to 0. 7935 7936 EVT VT = RV.getValueType(); 7937 7938 SDValue Zero = DAG.getConstantFP(0.0, VT.getScalarType()); 7939 if (VT.isVector()) { 7940 assert(VT.getVectorNumElements() == 4 && "Unknown vector type"); 7941 Zero = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Zero, Zero, Zero, Zero); 7942 } 7943 7944 SDValue ZeroCmp = 7945 DAG.getSetCC(dl, getSetCCResultType(*DAG.getContext(), VT), 7946 N->getOperand(0), Zero, ISD::SETEQ); 7947 DCI.AddToWorklist(ZeroCmp.getNode()); 7948 DCI.AddToWorklist(RV.getNode()); 7949 7950 RV = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, dl, VT, 7951 ZeroCmp, Zero, RV); 7952 return RV; 7953 } 7954 } 7955 7956 } 7957 break; 7958 case ISD::SINT_TO_FP: 7959 if (TM.getSubtarget<PPCSubtarget>().has64BitSupport()) { 7960 if (N->getOperand(0).getOpcode() == ISD::FP_TO_SINT) { 7961 // Turn (sint_to_fp (fp_to_sint X)) -> fctidz/fcfid without load/stores. 7962 // We allow the src/dst to be either f32/f64, but the intermediate 7963 // type must be i64. 7964 if (N->getOperand(0).getValueType() == MVT::i64 && 7965 N->getOperand(0).getOperand(0).getValueType() != MVT::ppcf128) { 7966 SDValue Val = N->getOperand(0).getOperand(0); 7967 if (Val.getValueType() == MVT::f32) { 7968 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 7969 DCI.AddToWorklist(Val.getNode()); 7970 } 7971 7972 Val = DAG.getNode(PPCISD::FCTIDZ, dl, MVT::f64, Val); 7973 DCI.AddToWorklist(Val.getNode()); 7974 Val = DAG.getNode(PPCISD::FCFID, dl, MVT::f64, Val); 7975 DCI.AddToWorklist(Val.getNode()); 7976 if (N->getValueType(0) == MVT::f32) { 7977 Val = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Val, 7978 DAG.getIntPtrConstant(0)); 7979 DCI.AddToWorklist(Val.getNode()); 7980 } 7981 return Val; 7982 } else if (N->getOperand(0).getValueType() == MVT::i32) { 7983 // If the intermediate type is i32, we can avoid the load/store here 7984 // too. 7985 } 7986 } 7987 } 7988 break; 7989 case ISD::STORE: 7990 // Turn STORE (FP_TO_SINT F) -> STFIWX(FCTIWZ(F)). 7991 if (TM.getSubtarget<PPCSubtarget>().hasSTFIWX() && 7992 !cast<StoreSDNode>(N)->isTruncatingStore() && 7993 N->getOperand(1).getOpcode() == ISD::FP_TO_SINT && 7994 N->getOperand(1).getValueType() == MVT::i32 && 7995 N->getOperand(1).getOperand(0).getValueType() != MVT::ppcf128) { 7996 SDValue Val = N->getOperand(1).getOperand(0); 7997 if (Val.getValueType() == MVT::f32) { 7998 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 7999 DCI.AddToWorklist(Val.getNode()); 8000 } 8001 Val = DAG.getNode(PPCISD::FCTIWZ, dl, MVT::f64, Val); 8002 DCI.AddToWorklist(Val.getNode()); 8003 8004 SDValue Ops[] = { 8005 N->getOperand(0), Val, N->getOperand(2), 8006 DAG.getValueType(N->getOperand(1).getValueType()) 8007 }; 8008 8009 Val = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 8010 DAG.getVTList(MVT::Other), Ops, 8011 cast<StoreSDNode>(N)->getMemoryVT(), 8012 cast<StoreSDNode>(N)->getMemOperand()); 8013 DCI.AddToWorklist(Val.getNode()); 8014 return Val; 8015 } 8016 8017 // Turn STORE (BSWAP) -> sthbrx/stwbrx. 8018 if (cast<StoreSDNode>(N)->isUnindexed() && 8019 N->getOperand(1).getOpcode() == ISD::BSWAP && 8020 N->getOperand(1).getNode()->hasOneUse() && 8021 (N->getOperand(1).getValueType() == MVT::i32 || 8022 N->getOperand(1).getValueType() == MVT::i16 || 8023 (TM.getSubtarget<PPCSubtarget>().hasLDBRX() && 8024 TM.getSubtarget<PPCSubtarget>().isPPC64() && 8025 N->getOperand(1).getValueType() == MVT::i64))) { 8026 SDValue BSwapOp = N->getOperand(1).getOperand(0); 8027 // Do an any-extend to 32-bits if this is a half-word input. 8028 if (BSwapOp.getValueType() == MVT::i16) 8029 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); 8030 8031 SDValue Ops[] = { 8032 N->getOperand(0), BSwapOp, N->getOperand(2), 8033 DAG.getValueType(N->getOperand(1).getValueType()) 8034 }; 8035 return 8036 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), 8037 Ops, cast<StoreSDNode>(N)->getMemoryVT(), 8038 cast<StoreSDNode>(N)->getMemOperand()); 8039 } 8040 break; 8041 case ISD::LOAD: { 8042 LoadSDNode *LD = cast<LoadSDNode>(N); 8043 EVT VT = LD->getValueType(0); 8044 Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 8045 unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(Ty); 8046 if (ISD::isNON_EXTLoad(N) && VT.isVector() && 8047 TM.getSubtarget<PPCSubtarget>().hasAltivec() && 8048 (VT == MVT::v16i8 || VT == MVT::v8i16 || 8049 VT == MVT::v4i32 || VT == MVT::v4f32) && 8050 LD->getAlignment() < ABIAlignment) { 8051 // This is a type-legal unaligned Altivec load. 8052 SDValue Chain = LD->getChain(); 8053 SDValue Ptr = LD->getBasePtr(); 8054 8055 // This implements the loading of unaligned vectors as described in 8056 // the venerable Apple Velocity Engine overview. Specifically: 8057 // https://developer.apple.com/hardwaredrivers/ve/alignment.html 8058 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html 8059 // 8060 // The general idea is to expand a sequence of one or more unaligned 8061 // loads into a alignment-based permutation-control instruction (lvsl), 8062 // a series of regular vector loads (which always truncate their 8063 // input address to an aligned address), and a series of permutations. 8064 // The results of these permutations are the requested loaded values. 8065 // The trick is that the last "extra" load is not taken from the address 8066 // you might suspect (sizeof(vector) bytes after the last requested 8067 // load), but rather sizeof(vector) - 1 bytes after the last 8068 // requested vector. The point of this is to avoid a page fault if the 8069 // base address happened to be aligned. This works because if the base 8070 // address is aligned, then adding less than a full vector length will 8071 // cause the last vector in the sequence to be (re)loaded. Otherwise, 8072 // the next vector will be fetched as you might suspect was necessary. 8073 8074 // We might be able to reuse the permutation generation from 8075 // a different base address offset from this one by an aligned amount. 8076 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this 8077 // optimization later. 8078 SDValue PermCntl = BuildIntrinsicOp(Intrinsic::ppc_altivec_lvsl, Ptr, 8079 DAG, dl, MVT::v16i8); 8080 8081 // Refine the alignment of the original load (a "new" load created here 8082 // which was identical to the first except for the alignment would be 8083 // merged with the existing node regardless). 8084 MachineFunction &MF = DAG.getMachineFunction(); 8085 MachineMemOperand *MMO = 8086 MF.getMachineMemOperand(LD->getPointerInfo(), 8087 LD->getMemOperand()->getFlags(), 8088 LD->getMemoryVT().getStoreSize(), 8089 ABIAlignment); 8090 LD->refineAlignment(MMO); 8091 SDValue BaseLoad = SDValue(LD, 0); 8092 8093 // Note that the value of IncOffset (which is provided to the next 8094 // load's pointer info offset value, and thus used to calculate the 8095 // alignment), and the value of IncValue (which is actually used to 8096 // increment the pointer value) are different! This is because we 8097 // require the next load to appear to be aligned, even though it 8098 // is actually offset from the base pointer by a lesser amount. 8099 int IncOffset = VT.getSizeInBits() / 8; 8100 int IncValue = IncOffset; 8101 8102 // Walk (both up and down) the chain looking for another load at the real 8103 // (aligned) offset (the alignment of the other load does not matter in 8104 // this case). If found, then do not use the offset reduction trick, as 8105 // that will prevent the loads from being later combined (as they would 8106 // otherwise be duplicates). 8107 if (!findConsecutiveLoad(LD, DAG)) 8108 --IncValue; 8109 8110 SDValue Increment = DAG.getConstant(IncValue, getPointerTy()); 8111 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 8112 8113 SDValue ExtraLoad = 8114 DAG.getLoad(VT, dl, Chain, Ptr, 8115 LD->getPointerInfo().getWithOffset(IncOffset), 8116 LD->isVolatile(), LD->isNonTemporal(), 8117 LD->isInvariant(), ABIAlignment); 8118 8119 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8120 BaseLoad.getValue(1), ExtraLoad.getValue(1)); 8121 8122 if (BaseLoad.getValueType() != MVT::v4i32) 8123 BaseLoad = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, BaseLoad); 8124 8125 if (ExtraLoad.getValueType() != MVT::v4i32) 8126 ExtraLoad = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, ExtraLoad); 8127 8128 SDValue Perm = BuildIntrinsicOp(Intrinsic::ppc_altivec_vperm, 8129 BaseLoad, ExtraLoad, PermCntl, DAG, dl); 8130 8131 if (VT != MVT::v4i32) 8132 Perm = DAG.getNode(ISD::BITCAST, dl, VT, Perm); 8133 8134 // Now we need to be really careful about how we update the users of the 8135 // original load. We cannot just call DCI.CombineTo (or 8136 // DAG.ReplaceAllUsesWith for that matter), because the load still has 8137 // uses created here (the permutation for example) that need to stay. 8138 SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 8139 while (UI != UE) { 8140 SDUse &Use = UI.getUse(); 8141 SDNode *User = *UI; 8142 // Note: BaseLoad is checked here because it might not be N, but a 8143 // bitcast of N. 8144 if (User == Perm.getNode() || User == BaseLoad.getNode() || 8145 User == TF.getNode() || Use.getResNo() > 1) { 8146 ++UI; 8147 continue; 8148 } 8149 8150 SDValue To = Use.getResNo() ? TF : Perm; 8151 ++UI; 8152 8153 SmallVector<SDValue, 8> Ops; 8154 for (SDNode::op_iterator O = User->op_begin(), 8155 OE = User->op_end(); O != OE; ++O) { 8156 if (*O == Use) 8157 Ops.push_back(To); 8158 else 8159 Ops.push_back(*O); 8160 } 8161 8162 DAG.UpdateNodeOperands(User, Ops); 8163 } 8164 8165 return SDValue(N, 0); 8166 } 8167 } 8168 break; 8169 case ISD::INTRINSIC_WO_CHAIN: 8170 if (cast<ConstantSDNode>(N->getOperand(0))->getZExtValue() == 8171 Intrinsic::ppc_altivec_lvsl && 8172 N->getOperand(1)->getOpcode() == ISD::ADD) { 8173 SDValue Add = N->getOperand(1); 8174 8175 if (DAG.MaskedValueIsZero(Add->getOperand(1), 8176 APInt::getAllOnesValue(4 /* 16 byte alignment */).zext( 8177 Add.getValueType().getScalarType().getSizeInBits()))) { 8178 SDNode *BasePtr = Add->getOperand(0).getNode(); 8179 for (SDNode::use_iterator UI = BasePtr->use_begin(), 8180 UE = BasePtr->use_end(); UI != UE; ++UI) { 8181 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 8182 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == 8183 Intrinsic::ppc_altivec_lvsl) { 8184 // We've found another LVSL, and this address if an aligned 8185 // multiple of that one. The results will be the same, so use the 8186 // one we've just found instead. 8187 8188 return SDValue(*UI, 0); 8189 } 8190 } 8191 } 8192 } 8193 8194 break; 8195 case ISD::BSWAP: 8196 // Turn BSWAP (LOAD) -> lhbrx/lwbrx. 8197 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && 8198 N->getOperand(0).hasOneUse() && 8199 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || 8200 (TM.getSubtarget<PPCSubtarget>().hasLDBRX() && 8201 TM.getSubtarget<PPCSubtarget>().isPPC64() && 8202 N->getValueType(0) == MVT::i64))) { 8203 SDValue Load = N->getOperand(0); 8204 LoadSDNode *LD = cast<LoadSDNode>(Load); 8205 // Create the byte-swapping load. 8206 SDValue Ops[] = { 8207 LD->getChain(), // Chain 8208 LD->getBasePtr(), // Ptr 8209 DAG.getValueType(N->getValueType(0)) // VT 8210 }; 8211 SDValue BSLoad = 8212 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, 8213 DAG.getVTList(N->getValueType(0) == MVT::i64 ? 8214 MVT::i64 : MVT::i32, MVT::Other), 8215 Ops, LD->getMemoryVT(), LD->getMemOperand()); 8216 8217 // If this is an i16 load, insert the truncate. 8218 SDValue ResVal = BSLoad; 8219 if (N->getValueType(0) == MVT::i16) 8220 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); 8221 8222 // First, combine the bswap away. This makes the value produced by the 8223 // load dead. 8224 DCI.CombineTo(N, ResVal); 8225 8226 // Next, combine the load away, we give it a bogus result value but a real 8227 // chain result. The result value is dead because the bswap is dead. 8228 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 8229 8230 // Return N so it doesn't get rechecked! 8231 return SDValue(N, 0); 8232 } 8233 8234 break; 8235 case PPCISD::VCMP: { 8236 // If a VCMPo node already exists with exactly the same operands as this 8237 // node, use its result instead of this node (VCMPo computes both a CR6 and 8238 // a normal output). 8239 // 8240 if (!N->getOperand(0).hasOneUse() && 8241 !N->getOperand(1).hasOneUse() && 8242 !N->getOperand(2).hasOneUse()) { 8243 8244 // Scan all of the users of the LHS, looking for VCMPo's that match. 8245 SDNode *VCMPoNode = nullptr; 8246 8247 SDNode *LHSN = N->getOperand(0).getNode(); 8248 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); 8249 UI != E; ++UI) 8250 if (UI->getOpcode() == PPCISD::VCMPo && 8251 UI->getOperand(1) == N->getOperand(1) && 8252 UI->getOperand(2) == N->getOperand(2) && 8253 UI->getOperand(0) == N->getOperand(0)) { 8254 VCMPoNode = *UI; 8255 break; 8256 } 8257 8258 // If there is no VCMPo node, or if the flag value has a single use, don't 8259 // transform this. 8260 if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) 8261 break; 8262 8263 // Look at the (necessarily single) use of the flag value. If it has a 8264 // chain, this transformation is more complex. Note that multiple things 8265 // could use the value result, which we should ignore. 8266 SDNode *FlagUser = nullptr; 8267 for (SDNode::use_iterator UI = VCMPoNode->use_begin(); 8268 FlagUser == nullptr; ++UI) { 8269 assert(UI != VCMPoNode->use_end() && "Didn't find user!"); 8270 SDNode *User = *UI; 8271 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 8272 if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { 8273 FlagUser = User; 8274 break; 8275 } 8276 } 8277 } 8278 8279 // If the user is a MFOCRF instruction, we know this is safe. 8280 // Otherwise we give up for right now. 8281 if (FlagUser->getOpcode() == PPCISD::MFOCRF) 8282 return SDValue(VCMPoNode, 0); 8283 } 8284 break; 8285 } 8286 case ISD::BRCOND: { 8287 SDValue Cond = N->getOperand(1); 8288 SDValue Target = N->getOperand(2); 8289 8290 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && 8291 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() == 8292 Intrinsic::ppc_is_decremented_ctr_nonzero) { 8293 8294 // We now need to make the intrinsic dead (it cannot be instruction 8295 // selected). 8296 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); 8297 assert(Cond.getNode()->hasOneUse() && 8298 "Counter decrement has more than one use"); 8299 8300 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, 8301 N->getOperand(0), Target); 8302 } 8303 } 8304 break; 8305 case ISD::BR_CC: { 8306 // If this is a branch on an altivec predicate comparison, lower this so 8307 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This 8308 // lowering is done pre-legalize, because the legalizer lowers the predicate 8309 // compare down to code that is difficult to reassemble. 8310 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 8311 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); 8312 8313 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero 8314 // value. If so, pass-through the AND to get to the intrinsic. 8315 if (LHS.getOpcode() == ISD::AND && 8316 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && 8317 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() == 8318 Intrinsic::ppc_is_decremented_ctr_nonzero && 8319 isa<ConstantSDNode>(LHS.getOperand(1)) && 8320 !cast<ConstantSDNode>(LHS.getOperand(1))->getConstantIntValue()-> 8321 isZero()) 8322 LHS = LHS.getOperand(0); 8323 8324 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && 8325 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() == 8326 Intrinsic::ppc_is_decremented_ctr_nonzero && 8327 isa<ConstantSDNode>(RHS)) { 8328 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 8329 "Counter decrement comparison is not EQ or NE"); 8330 8331 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 8332 bool isBDNZ = (CC == ISD::SETEQ && Val) || 8333 (CC == ISD::SETNE && !Val); 8334 8335 // We now need to make the intrinsic dead (it cannot be instruction 8336 // selected). 8337 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); 8338 assert(LHS.getNode()->hasOneUse() && 8339 "Counter decrement has more than one use"); 8340 8341 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, 8342 N->getOperand(0), N->getOperand(4)); 8343 } 8344 8345 int CompareOpc; 8346 bool isDot; 8347 8348 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 8349 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && 8350 getAltivecCompareInfo(LHS, CompareOpc, isDot)) { 8351 assert(isDot && "Can't compare against a vector result!"); 8352 8353 // If this is a comparison against something other than 0/1, then we know 8354 // that the condition is never/always true. 8355 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 8356 if (Val != 0 && Val != 1) { 8357 if (CC == ISD::SETEQ) // Cond never true, remove branch. 8358 return N->getOperand(0); 8359 // Always !=, turn it into an unconditional branch. 8360 return DAG.getNode(ISD::BR, dl, MVT::Other, 8361 N->getOperand(0), N->getOperand(4)); 8362 } 8363 8364 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); 8365 8366 // Create the PPCISD altivec 'dot' comparison node. 8367 SDValue Ops[] = { 8368 LHS.getOperand(2), // LHS of compare 8369 LHS.getOperand(3), // RHS of compare 8370 DAG.getConstant(CompareOpc, MVT::i32) 8371 }; 8372 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; 8373 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 8374 8375 // Unpack the result based on how the target uses it. 8376 PPC::Predicate CompOpc; 8377 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) { 8378 default: // Can't happen, don't crash on invalid number though. 8379 case 0: // Branch on the value of the EQ bit of CR6. 8380 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; 8381 break; 8382 case 1: // Branch on the inverted value of the EQ bit of CR6. 8383 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; 8384 break; 8385 case 2: // Branch on the value of the LT bit of CR6. 8386 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; 8387 break; 8388 case 3: // Branch on the inverted value of the LT bit of CR6. 8389 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; 8390 break; 8391 } 8392 8393 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), 8394 DAG.getConstant(CompOpc, MVT::i32), 8395 DAG.getRegister(PPC::CR6, MVT::i32), 8396 N->getOperand(4), CompNode.getValue(1)); 8397 } 8398 break; 8399 } 8400 } 8401 8402 return SDValue(); 8403} 8404 8405//===----------------------------------------------------------------------===// 8406// Inline Assembly Support 8407//===----------------------------------------------------------------------===// 8408 8409void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 8410 APInt &KnownZero, 8411 APInt &KnownOne, 8412 const SelectionDAG &DAG, 8413 unsigned Depth) const { 8414 KnownZero = KnownOne = APInt(KnownZero.getBitWidth(), 0); 8415 switch (Op.getOpcode()) { 8416 default: break; 8417 case PPCISD::LBRX: { 8418 // lhbrx is known to have the top bits cleared out. 8419 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16) 8420 KnownZero = 0xFFFF0000; 8421 break; 8422 } 8423 case ISD::INTRINSIC_WO_CHAIN: { 8424 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 8425 default: break; 8426 case Intrinsic::ppc_altivec_vcmpbfp_p: 8427 case Intrinsic::ppc_altivec_vcmpeqfp_p: 8428 case Intrinsic::ppc_altivec_vcmpequb_p: 8429 case Intrinsic::ppc_altivec_vcmpequh_p: 8430 case Intrinsic::ppc_altivec_vcmpequw_p: 8431 case Intrinsic::ppc_altivec_vcmpgefp_p: 8432 case Intrinsic::ppc_altivec_vcmpgtfp_p: 8433 case Intrinsic::ppc_altivec_vcmpgtsb_p: 8434 case Intrinsic::ppc_altivec_vcmpgtsh_p: 8435 case Intrinsic::ppc_altivec_vcmpgtsw_p: 8436 case Intrinsic::ppc_altivec_vcmpgtub_p: 8437 case Intrinsic::ppc_altivec_vcmpgtuh_p: 8438 case Intrinsic::ppc_altivec_vcmpgtuw_p: 8439 KnownZero = ~1U; // All bits but the low one are known to be zero. 8440 break; 8441 } 8442 } 8443 } 8444} 8445 8446 8447/// getConstraintType - Given a constraint, return the type of 8448/// constraint it is for this target. 8449PPCTargetLowering::ConstraintType 8450PPCTargetLowering::getConstraintType(const std::string &Constraint) const { 8451 if (Constraint.size() == 1) { 8452 switch (Constraint[0]) { 8453 default: break; 8454 case 'b': 8455 case 'r': 8456 case 'f': 8457 case 'v': 8458 case 'y': 8459 return C_RegisterClass; 8460 case 'Z': 8461 // FIXME: While Z does indicate a memory constraint, it specifically 8462 // indicates an r+r address (used in conjunction with the 'y' modifier 8463 // in the replacement string). Currently, we're forcing the base 8464 // register to be r0 in the asm printer (which is interpreted as zero) 8465 // and forming the complete address in the second register. This is 8466 // suboptimal. 8467 return C_Memory; 8468 } 8469 } else if (Constraint == "wc") { // individual CR bits. 8470 return C_RegisterClass; 8471 } else if (Constraint == "wa" || Constraint == "wd" || 8472 Constraint == "wf" || Constraint == "ws") { 8473 return C_RegisterClass; // VSX registers. 8474 } 8475 return TargetLowering::getConstraintType(Constraint); 8476} 8477 8478/// Examine constraint type and operand type and determine a weight value. 8479/// This object must already have been set up with the operand type 8480/// and the current alternative constraint selected. 8481TargetLowering::ConstraintWeight 8482PPCTargetLowering::getSingleConstraintMatchWeight( 8483 AsmOperandInfo &info, const char *constraint) const { 8484 ConstraintWeight weight = CW_Invalid; 8485 Value *CallOperandVal = info.CallOperandVal; 8486 // If we don't have a value, we can't do a match, 8487 // but allow it at the lowest weight. 8488 if (!CallOperandVal) 8489 return CW_Default; 8490 Type *type = CallOperandVal->getType(); 8491 8492 // Look at the constraint type. 8493 if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) 8494 return CW_Register; // an individual CR bit. 8495 else if ((StringRef(constraint) == "wa" || 8496 StringRef(constraint) == "wd" || 8497 StringRef(constraint) == "wf") && 8498 type->isVectorTy()) 8499 return CW_Register; 8500 else if (StringRef(constraint) == "ws" && type->isDoubleTy()) 8501 return CW_Register; 8502 8503 switch (*constraint) { 8504 default: 8505 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 8506 break; 8507 case 'b': 8508 if (type->isIntegerTy()) 8509 weight = CW_Register; 8510 break; 8511 case 'f': 8512 if (type->isFloatTy()) 8513 weight = CW_Register; 8514 break; 8515 case 'd': 8516 if (type->isDoubleTy()) 8517 weight = CW_Register; 8518 break; 8519 case 'v': 8520 if (type->isVectorTy()) 8521 weight = CW_Register; 8522 break; 8523 case 'y': 8524 weight = CW_Register; 8525 break; 8526 case 'Z': 8527 weight = CW_Memory; 8528 break; 8529 } 8530 return weight; 8531} 8532 8533std::pair<unsigned, const TargetRegisterClass*> 8534PPCTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 8535 MVT VT) const { 8536 if (Constraint.size() == 1) { 8537 // GCC RS6000 Constraint Letters 8538 switch (Constraint[0]) { 8539 case 'b': // R1-R31 8540 if (VT == MVT::i64 && PPCSubTarget.isPPC64()) 8541 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); 8542 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); 8543 case 'r': // R0-R31 8544 if (VT == MVT::i64 && PPCSubTarget.isPPC64()) 8545 return std::make_pair(0U, &PPC::G8RCRegClass); 8546 return std::make_pair(0U, &PPC::GPRCRegClass); 8547 case 'f': 8548 if (VT == MVT::f32 || VT == MVT::i32) 8549 return std::make_pair(0U, &PPC::F4RCRegClass); 8550 if (VT == MVT::f64 || VT == MVT::i64) 8551 return std::make_pair(0U, &PPC::F8RCRegClass); 8552 break; 8553 case 'v': 8554 return std::make_pair(0U, &PPC::VRRCRegClass); 8555 case 'y': // crrc 8556 return std::make_pair(0U, &PPC::CRRCRegClass); 8557 } 8558 } else if (Constraint == "wc") { // an individual CR bit. 8559 return std::make_pair(0U, &PPC::CRBITRCRegClass); 8560 } else if (Constraint == "wa" || Constraint == "wd" || 8561 Constraint == "wf") { 8562 return std::make_pair(0U, &PPC::VSRCRegClass); 8563 } else if (Constraint == "ws") { 8564 return std::make_pair(0U, &PPC::VSFRCRegClass); 8565 } 8566 8567 std::pair<unsigned, const TargetRegisterClass*> R = 8568 TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 8569 8570 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers 8571 // (which we call X[0-9]+). If a 64-bit value has been requested, and a 8572 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent 8573 // register. 8574 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use 8575 // the AsmName field from *RegisterInfo.td, then this would not be necessary. 8576 if (R.first && VT == MVT::i64 && PPCSubTarget.isPPC64() && 8577 PPC::GPRCRegClass.contains(R.first)) { 8578 const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo(); 8579 return std::make_pair(TRI->getMatchingSuperReg(R.first, 8580 PPC::sub_32, &PPC::G8RCRegClass), 8581 &PPC::G8RCRegClass); 8582 } 8583 8584 return R; 8585} 8586 8587 8588/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 8589/// vector. If it is invalid, don't add anything to Ops. 8590void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 8591 std::string &Constraint, 8592 std::vector<SDValue>&Ops, 8593 SelectionDAG &DAG) const { 8594 SDValue Result; 8595 8596 // Only support length 1 constraints. 8597 if (Constraint.length() > 1) return; 8598 8599 char Letter = Constraint[0]; 8600 switch (Letter) { 8601 default: break; 8602 case 'I': 8603 case 'J': 8604 case 'K': 8605 case 'L': 8606 case 'M': 8607 case 'N': 8608 case 'O': 8609 case 'P': { 8610 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op); 8611 if (!CST) return; // Must be an immediate to match. 8612 unsigned Value = CST->getZExtValue(); 8613 switch (Letter) { 8614 default: llvm_unreachable("Unknown constraint letter!"); 8615 case 'I': // "I" is a signed 16-bit constant. 8616 if ((short)Value == (int)Value) 8617 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8618 break; 8619 case 'J': // "J" is a constant with only the high-order 16 bits nonzero. 8620 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. 8621 if ((short)Value == 0) 8622 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8623 break; 8624 case 'K': // "K" is a constant with only the low-order 16 bits nonzero. 8625 if ((Value >> 16) == 0) 8626 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8627 break; 8628 case 'M': // "M" is a constant that is greater than 31. 8629 if (Value > 31) 8630 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8631 break; 8632 case 'N': // "N" is a positive constant that is an exact power of two. 8633 if ((int)Value > 0 && isPowerOf2_32(Value)) 8634 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8635 break; 8636 case 'O': // "O" is the constant zero. 8637 if (Value == 0) 8638 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8639 break; 8640 case 'P': // "P" is a constant whose negation is a signed 16-bit constant. 8641 if ((short)-Value == (int)-Value) 8642 Result = DAG.getTargetConstant(Value, Op.getValueType()); 8643 break; 8644 } 8645 break; 8646 } 8647 } 8648 8649 if (Result.getNode()) { 8650 Ops.push_back(Result); 8651 return; 8652 } 8653 8654 // Handle standard constraint letters. 8655 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 8656} 8657 8658// isLegalAddressingMode - Return true if the addressing mode represented 8659// by AM is legal for this target, for a load/store of the specified type. 8660bool PPCTargetLowering::isLegalAddressingMode(const AddrMode &AM, 8661 Type *Ty) const { 8662 // FIXME: PPC does not allow r+i addressing modes for vectors! 8663 8664 // PPC allows a sign-extended 16-bit immediate field. 8665 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 8666 return false; 8667 8668 // No global is ever allowed as a base. 8669 if (AM.BaseGV) 8670 return false; 8671 8672 // PPC only support r+r, 8673 switch (AM.Scale) { 8674 case 0: // "r+i" or just "i", depending on HasBaseReg. 8675 break; 8676 case 1: 8677 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 8678 return false; 8679 // Otherwise we have r+r or r+i. 8680 break; 8681 case 2: 8682 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 8683 return false; 8684 // Allow 2*r as r+r. 8685 break; 8686 default: 8687 // No other scales are supported. 8688 return false; 8689 } 8690 8691 return true; 8692} 8693 8694SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, 8695 SelectionDAG &DAG) const { 8696 MachineFunction &MF = DAG.getMachineFunction(); 8697 MachineFrameInfo *MFI = MF.getFrameInfo(); 8698 MFI->setReturnAddressIsTaken(true); 8699 8700 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 8701 return SDValue(); 8702 8703 SDLoc dl(Op); 8704 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 8705 8706 // Make sure the function does not optimize away the store of the RA to 8707 // the stack. 8708 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 8709 FuncInfo->setLRStoreRequired(); 8710 bool isPPC64 = PPCSubTarget.isPPC64(); 8711 bool isDarwinABI = PPCSubTarget.isDarwinABI(); 8712 8713 if (Depth > 0) { 8714 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 8715 SDValue Offset = 8716 8717 DAG.getConstant(PPCFrameLowering::getReturnSaveOffset(isPPC64, isDarwinABI), 8718 isPPC64? MVT::i64 : MVT::i32); 8719 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 8720 DAG.getNode(ISD::ADD, dl, getPointerTy(), 8721 FrameAddr, Offset), 8722 MachinePointerInfo(), false, false, false, 0); 8723 } 8724 8725 // Just load the return address off the stack. 8726 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); 8727 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 8728 RetAddrFI, MachinePointerInfo(), false, false, false, 0); 8729} 8730 8731SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, 8732 SelectionDAG &DAG) const { 8733 SDLoc dl(Op); 8734 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 8735 8736 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(); 8737 bool isPPC64 = PtrVT == MVT::i64; 8738 8739 MachineFunction &MF = DAG.getMachineFunction(); 8740 MachineFrameInfo *MFI = MF.getFrameInfo(); 8741 MFI->setFrameAddressIsTaken(true); 8742 8743 // Naked functions never have a frame pointer, and so we use r1. For all 8744 // other functions, this decision must be delayed until during PEI. 8745 unsigned FrameReg; 8746 if (MF.getFunction()->getAttributes().hasAttribute( 8747 AttributeSet::FunctionIndex, Attribute::Naked)) 8748 FrameReg = isPPC64 ? PPC::X1 : PPC::R1; 8749 else 8750 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; 8751 8752 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, 8753 PtrVT); 8754 while (Depth--) 8755 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 8756 FrameAddr, MachinePointerInfo(), false, false, 8757 false, 0); 8758 return FrameAddr; 8759} 8760 8761// FIXME? Maybe this could be a TableGen attribute on some registers and 8762// this table could be generated automatically from RegInfo. 8763unsigned PPCTargetLowering::getRegisterByName(const char* RegName, 8764 EVT VT) const { 8765 bool isPPC64 = PPCSubTarget.isPPC64(); 8766 bool isDarwinABI = PPCSubTarget.isDarwinABI(); 8767 8768 if ((isPPC64 && VT != MVT::i64 && VT != MVT::i32) || 8769 (!isPPC64 && VT != MVT::i32)) 8770 report_fatal_error("Invalid register global variable type"); 8771 8772 bool is64Bit = isPPC64 && VT == MVT::i64; 8773 unsigned Reg = StringSwitch<unsigned>(RegName) 8774 .Case("r1", is64Bit ? PPC::X1 : PPC::R1) 8775 .Case("r2", isDarwinABI ? 0 : (is64Bit ? PPC::X2 : PPC::R2)) 8776 .Case("r13", (!isPPC64 && isDarwinABI) ? 0 : 8777 (is64Bit ? PPC::X13 : PPC::R13)) 8778 .Default(0); 8779 8780 if (Reg) 8781 return Reg; 8782 report_fatal_error("Invalid register name global variable"); 8783} 8784 8785bool 8786PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 8787 // The PowerPC target isn't yet aware of offsets. 8788 return false; 8789} 8790 8791/// getOptimalMemOpType - Returns the target specific optimal type for load 8792/// and store operations as a result of memset, memcpy, and memmove 8793/// lowering. If DstAlign is zero that means it's safe to destination 8794/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 8795/// means there isn't a need to check it against alignment requirement, 8796/// probably because the source does not need to be loaded. If 'IsMemset' is 8797/// true, that means it's expanding a memset. If 'ZeroMemset' is true, that 8798/// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy 8799/// source is constant so it does not need to be loaded. 8800/// It returns EVT::Other if the type should be determined using generic 8801/// target-independent logic. 8802EVT PPCTargetLowering::getOptimalMemOpType(uint64_t Size, 8803 unsigned DstAlign, unsigned SrcAlign, 8804 bool IsMemset, bool ZeroMemset, 8805 bool MemcpyStrSrc, 8806 MachineFunction &MF) const { 8807 if (this->PPCSubTarget.isPPC64()) { 8808 return MVT::i64; 8809 } else { 8810 return MVT::i32; 8811 } 8812} 8813 8814/// \brief Returns true if it is beneficial to convert a load of a constant 8815/// to just the constant itself. 8816bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 8817 Type *Ty) const { 8818 assert(Ty->isIntegerTy()); 8819 8820 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 8821 if (BitSize == 0 || BitSize > 64) 8822 return false; 8823 return true; 8824} 8825 8826bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 8827 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 8828 return false; 8829 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 8830 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 8831 return NumBits1 == 64 && NumBits2 == 32; 8832} 8833 8834bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 8835 if (!VT1.isInteger() || !VT2.isInteger()) 8836 return false; 8837 unsigned NumBits1 = VT1.getSizeInBits(); 8838 unsigned NumBits2 = VT2.getSizeInBits(); 8839 return NumBits1 == 64 && NumBits2 == 32; 8840} 8841 8842bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 8843 return isInt<16>(Imm) || isUInt<16>(Imm); 8844} 8845 8846bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { 8847 return isInt<16>(Imm) || isUInt<16>(Imm); 8848} 8849 8850bool PPCTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 8851 unsigned, 8852 bool *Fast) const { 8853 if (DisablePPCUnaligned) 8854 return false; 8855 8856 // PowerPC supports unaligned memory access for simple non-vector types. 8857 // Although accessing unaligned addresses is not as efficient as accessing 8858 // aligned addresses, it is generally more efficient than manual expansion, 8859 // and generally only traps for software emulation when crossing page 8860 // boundaries. 8861 8862 if (!VT.isSimple()) 8863 return false; 8864 8865 if (VT.getSimpleVT().isVector()) { 8866 if (PPCSubTarget.hasVSX()) { 8867 if (VT != MVT::v2f64 && VT != MVT::v2i64) 8868 return false; 8869 } else { 8870 return false; 8871 } 8872 } 8873 8874 if (VT == MVT::ppcf128) 8875 return false; 8876 8877 if (Fast) 8878 *Fast = true; 8879 8880 return true; 8881} 8882 8883bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 8884 VT = VT.getScalarType(); 8885 8886 if (!VT.isSimple()) 8887 return false; 8888 8889 switch (VT.getSimpleVT().SimpleTy) { 8890 case MVT::f32: 8891 case MVT::f64: 8892 return true; 8893 default: 8894 break; 8895 } 8896 8897 return false; 8898} 8899 8900bool 8901PPCTargetLowering::shouldExpandBuildVectorWithShuffles( 8902 EVT VT , unsigned DefinedValues) const { 8903 if (VT == MVT::v2i64) 8904 return false; 8905 8906 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); 8907} 8908 8909Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { 8910 if (DisableILPPref || PPCSubTarget.enableMachineScheduler()) 8911 return TargetLowering::getSchedulingPreference(N); 8912 8913 return Sched::ILP; 8914} 8915 8916// Create a fast isel object. 8917FastISel * 8918PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, 8919 const TargetLibraryInfo *LibInfo) const { 8920 return PPC::createFastISel(FuncInfo, LibInfo); 8921} 8922