SystemZISelLowering.cpp revision 35c93e4e42d7a35a90e89211fa62f478e25ba0a4
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "systemz-lower" 15 16#include "SystemZISelLowering.h" 17#include "SystemZCallingConv.h" 18#include "SystemZConstantPoolValue.h" 19#include "SystemZMachineFunctionInfo.h" 20#include "SystemZTargetMachine.h" 21#include "llvm/CodeGen/CallingConvLower.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25 26using namespace llvm; 27 28// Classify VT as either 32 or 64 bit. 29static bool is32Bit(EVT VT) { 30 switch (VT.getSimpleVT().SimpleTy) { 31 case MVT::i32: 32 return true; 33 case MVT::i64: 34 return false; 35 default: 36 llvm_unreachable("Unsupported type"); 37 } 38} 39 40// Return a version of MachineOperand that can be safely used before the 41// final use. 42static MachineOperand earlyUseOperand(MachineOperand Op) { 43 if (Op.isReg()) 44 Op.setIsKill(false); 45 return Op; 46} 47 48SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 49 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 50 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 51 MVT PtrVT = getPointerTy(); 52 53 // Set up the register classes. 54 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 55 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 56 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 57 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 58 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 59 60 // Compute derived properties from the register classes 61 computeRegisterProperties(); 62 63 // Set up special registers. 64 setExceptionPointerRegister(SystemZ::R6D); 65 setExceptionSelectorRegister(SystemZ::R7D); 66 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 67 68 // TODO: It may be better to default to latency-oriented scheduling, however 69 // LLVM's current latency-oriented scheduler can't handle physreg definitions 70 // such as SystemZ has with CC, so set this to the register-pressure 71 // scheduler, because it can. 72 setSchedulingPreference(Sched::RegPressure); 73 74 setBooleanContents(ZeroOrOneBooleanContent); 75 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 76 77 // Instructions are strings of 2-byte aligned 2-byte values. 78 setMinFunctionAlignment(2); 79 80 // Handle operations that are handled in a similar way for all types. 81 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 82 I <= MVT::LAST_FP_VALUETYPE; 83 ++I) { 84 MVT VT = MVT::SimpleValueType(I); 85 if (isTypeLegal(VT)) { 86 // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). 87 setOperationAction(ISD::SETCC, VT, Expand); 88 89 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 90 setOperationAction(ISD::SELECT, VT, Expand); 91 92 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 93 setOperationAction(ISD::SELECT_CC, VT, Custom); 94 setOperationAction(ISD::BR_CC, VT, Custom); 95 } 96 } 97 98 // Expand jump table branches as address arithmetic followed by an 99 // indirect jump. 100 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 101 102 // Expand BRCOND into a BR_CC (see above). 103 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 104 105 // Handle integer types. 106 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 107 I <= MVT::LAST_INTEGER_VALUETYPE; 108 ++I) { 109 MVT VT = MVT::SimpleValueType(I); 110 if (isTypeLegal(VT)) { 111 // Expand individual DIV and REMs into DIVREMs. 112 setOperationAction(ISD::SDIV, VT, Expand); 113 setOperationAction(ISD::UDIV, VT, Expand); 114 setOperationAction(ISD::SREM, VT, Expand); 115 setOperationAction(ISD::UREM, VT, Expand); 116 setOperationAction(ISD::SDIVREM, VT, Custom); 117 setOperationAction(ISD::UDIVREM, VT, Custom); 118 119 // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 120 // FIXME: probably much too conservative. 121 setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 122 setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 123 124 // No special instructions for these. 125 setOperationAction(ISD::CTPOP, VT, Expand); 126 setOperationAction(ISD::CTTZ, VT, Expand); 127 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 128 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 129 setOperationAction(ISD::ROTR, VT, Expand); 130 131 // Use *MUL_LOHI where possible instead of MULH*. 132 setOperationAction(ISD::MULHS, VT, Expand); 133 setOperationAction(ISD::MULHU, VT, Expand); 134 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 135 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 136 137 // We have instructions for signed but not unsigned FP conversion. 138 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 139 } 140 } 141 142 // Type legalization will convert 8- and 16-bit atomic operations into 143 // forms that operate on i32s (but still keeping the original memory VT). 144 // Lower them into full i32 operations. 145 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 146 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 147 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 148 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 149 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 150 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 151 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 152 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 153 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 154 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 155 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 156 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 157 158 // We have instructions for signed but not unsigned FP conversion. 159 // Handle unsigned 32-bit types as signed 64-bit types. 160 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 161 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 162 163 // We have native support for a 64-bit CTLZ, via FLOGR. 164 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 165 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 166 167 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 168 setOperationAction(ISD::OR, MVT::i64, Custom); 169 170 // FIXME: Can we support these natively? 171 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 172 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 173 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 174 175 // We have native instructions for i8, i16 and i32 extensions, but not i1. 176 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 177 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 178 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 179 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 180 181 // Handle the various types of symbolic address. 182 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 183 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 184 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 185 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 186 setOperationAction(ISD::JumpTable, PtrVT, Custom); 187 188 // We need to handle dynamic allocations specially because of the 189 // 160-byte area at the bottom of the stack. 190 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 191 192 // Use custom expanders so that we can force the function to use 193 // a frame pointer. 194 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 195 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 196 197 // Handle floating-point types. 198 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 199 I <= MVT::LAST_FP_VALUETYPE; 200 ++I) { 201 MVT VT = MVT::SimpleValueType(I); 202 if (isTypeLegal(VT)) { 203 // We can use FI for FRINT. 204 setOperationAction(ISD::FRINT, VT, Legal); 205 206 // We can use the extended form of FI for other rounding operations. 207 if (Subtarget.hasFPExtension()) { 208 setOperationAction(ISD::FNEARBYINT, VT, Legal); 209 setOperationAction(ISD::FFLOOR, VT, Legal); 210 setOperationAction(ISD::FCEIL, VT, Legal); 211 setOperationAction(ISD::FTRUNC, VT, Legal); 212 setOperationAction(ISD::FROUND, VT, Legal); 213 } 214 215 // No special instructions for these. 216 setOperationAction(ISD::FSIN, VT, Expand); 217 setOperationAction(ISD::FCOS, VT, Expand); 218 setOperationAction(ISD::FREM, VT, Expand); 219 } 220 } 221 222 // We have fused multiply-addition for f32 and f64 but not f128. 223 setOperationAction(ISD::FMA, MVT::f32, Legal); 224 setOperationAction(ISD::FMA, MVT::f64, Legal); 225 setOperationAction(ISD::FMA, MVT::f128, Expand); 226 227 // Needed so that we don't try to implement f128 constant loads using 228 // a load-and-extend of a f80 constant (in cases where the constant 229 // would fit in an f80). 230 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 231 232 // Floating-point truncation and stores need to be done separately. 233 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 234 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 235 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 236 237 // We have 64-bit FPR<->GPR moves, but need special handling for 238 // 32-bit forms. 239 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 240 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 241 242 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 243 // structure, but VAEND is a no-op. 244 setOperationAction(ISD::VASTART, MVT::Other, Custom); 245 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 246 setOperationAction(ISD::VAEND, MVT::Other, Expand); 247 248 // We want to use MVC in preference to even a single load/store pair. 249 MaxStoresPerMemcpy = 0; 250 MaxStoresPerMemcpyOptSize = 0; 251 252 // The main memset sequence is a byte store followed by an MVC. 253 // Two STC or MV..I stores win over that, but the kind of fused stores 254 // generated by target-independent code don't when the byte value is 255 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 256 // than "STC;MVC". Handle the choice in target-specific code instead. 257 MaxStoresPerMemset = 0; 258 MaxStoresPerMemsetOptSize = 0; 259} 260 261bool 262SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 263 VT = VT.getScalarType(); 264 265 if (!VT.isSimple()) 266 return false; 267 268 switch (VT.getSimpleVT().SimpleTy) { 269 case MVT::f32: 270 case MVT::f64: 271 return true; 272 case MVT::f128: 273 return false; 274 default: 275 break; 276 } 277 278 return false; 279} 280 281bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 282 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 283 return Imm.isZero() || Imm.isNegZero(); 284} 285 286bool SystemZTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 287 bool *Fast) const { 288 // Unaligned accesses should never be slower than the expanded version. 289 // We check specifically for aligned accesses in the few cases where 290 // they are required. 291 if (Fast) 292 *Fast = true; 293 return true; 294} 295 296bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 297 Type *Ty) const { 298 // Punt on globals for now, although they can be used in limited 299 // RELATIVE LONG cases. 300 if (AM.BaseGV) 301 return false; 302 303 // Require a 20-bit signed offset. 304 if (!isInt<20>(AM.BaseOffs)) 305 return false; 306 307 // Indexing is OK but no scale factor can be applied. 308 return AM.Scale == 0 || AM.Scale == 1; 309} 310 311bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 312 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 313 return false; 314 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 315 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 316 return FromBits > ToBits; 317} 318 319bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 320 if (!FromVT.isInteger() || !ToVT.isInteger()) 321 return false; 322 unsigned FromBits = FromVT.getSizeInBits(); 323 unsigned ToBits = ToVT.getSizeInBits(); 324 return FromBits > ToBits; 325} 326 327//===----------------------------------------------------------------------===// 328// Inline asm support 329//===----------------------------------------------------------------------===// 330 331TargetLowering::ConstraintType 332SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 333 if (Constraint.size() == 1) { 334 switch (Constraint[0]) { 335 case 'a': // Address register 336 case 'd': // Data register (equivalent to 'r') 337 case 'f': // Floating-point register 338 case 'r': // General-purpose register 339 return C_RegisterClass; 340 341 case 'Q': // Memory with base and unsigned 12-bit displacement 342 case 'R': // Likewise, plus an index 343 case 'S': // Memory with base and signed 20-bit displacement 344 case 'T': // Likewise, plus an index 345 case 'm': // Equivalent to 'T'. 346 return C_Memory; 347 348 case 'I': // Unsigned 8-bit constant 349 case 'J': // Unsigned 12-bit constant 350 case 'K': // Signed 16-bit constant 351 case 'L': // Signed 20-bit displacement (on all targets we support) 352 case 'M': // 0x7fffffff 353 return C_Other; 354 355 default: 356 break; 357 } 358 } 359 return TargetLowering::getConstraintType(Constraint); 360} 361 362TargetLowering::ConstraintWeight SystemZTargetLowering:: 363getSingleConstraintMatchWeight(AsmOperandInfo &info, 364 const char *constraint) const { 365 ConstraintWeight weight = CW_Invalid; 366 Value *CallOperandVal = info.CallOperandVal; 367 // If we don't have a value, we can't do a match, 368 // but allow it at the lowest weight. 369 if (CallOperandVal == NULL) 370 return CW_Default; 371 Type *type = CallOperandVal->getType(); 372 // Look at the constraint type. 373 switch (*constraint) { 374 default: 375 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 376 break; 377 378 case 'a': // Address register 379 case 'd': // Data register (equivalent to 'r') 380 case 'r': // General-purpose register 381 if (CallOperandVal->getType()->isIntegerTy()) 382 weight = CW_Register; 383 break; 384 385 case 'f': // Floating-point register 386 if (type->isFloatingPointTy()) 387 weight = CW_Register; 388 break; 389 390 case 'I': // Unsigned 8-bit constant 391 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 392 if (isUInt<8>(C->getZExtValue())) 393 weight = CW_Constant; 394 break; 395 396 case 'J': // Unsigned 12-bit constant 397 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 398 if (isUInt<12>(C->getZExtValue())) 399 weight = CW_Constant; 400 break; 401 402 case 'K': // Signed 16-bit constant 403 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 404 if (isInt<16>(C->getSExtValue())) 405 weight = CW_Constant; 406 break; 407 408 case 'L': // Signed 20-bit displacement (on all targets we support) 409 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 410 if (isInt<20>(C->getSExtValue())) 411 weight = CW_Constant; 412 break; 413 414 case 'M': // 0x7fffffff 415 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 416 if (C->getZExtValue() == 0x7fffffff) 417 weight = CW_Constant; 418 break; 419 } 420 return weight; 421} 422 423// Parse a "{tNNN}" register constraint for which the register type "t" 424// has already been verified. MC is the class associated with "t" and 425// Map maps 0-based register numbers to LLVM register numbers. 426static std::pair<unsigned, const TargetRegisterClass *> 427parseRegisterNumber(const std::string &Constraint, 428 const TargetRegisterClass *RC, const unsigned *Map) { 429 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 430 if (isdigit(Constraint[2])) { 431 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 432 unsigned Index = atoi(Suffix.c_str()); 433 if (Index < 16 && Map[Index]) 434 return std::make_pair(Map[Index], RC); 435 } 436 return std::make_pair(0u, static_cast<TargetRegisterClass*>(0)); 437} 438 439std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 440getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 441 if (Constraint.size() == 1) { 442 // GCC Constraint Letters 443 switch (Constraint[0]) { 444 default: break; 445 case 'd': // Data register (equivalent to 'r') 446 case 'r': // General-purpose register 447 if (VT == MVT::i64) 448 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 449 else if (VT == MVT::i128) 450 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 451 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 452 453 case 'a': // Address register 454 if (VT == MVT::i64) 455 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 456 else if (VT == MVT::i128) 457 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 458 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 459 460 case 'f': // Floating-point register 461 if (VT == MVT::f64) 462 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 463 else if (VT == MVT::f128) 464 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 465 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 466 } 467 } 468 if (Constraint[0] == '{') { 469 // We need to override the default register parsing for GPRs and FPRs 470 // because the interpretation depends on VT. The internal names of 471 // the registers are also different from the external names 472 // (F0D and F0S instead of F0, etc.). 473 if (Constraint[1] == 'r') { 474 if (VT == MVT::i32) 475 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 476 SystemZMC::GR32Regs); 477 if (VT == MVT::i128) 478 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 479 SystemZMC::GR128Regs); 480 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 481 SystemZMC::GR64Regs); 482 } 483 if (Constraint[1] == 'f') { 484 if (VT == MVT::f32) 485 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 486 SystemZMC::FP32Regs); 487 if (VT == MVT::f128) 488 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 489 SystemZMC::FP128Regs); 490 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 491 SystemZMC::FP64Regs); 492 } 493 } 494 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 495} 496 497void SystemZTargetLowering:: 498LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 499 std::vector<SDValue> &Ops, 500 SelectionDAG &DAG) const { 501 // Only support length 1 constraints for now. 502 if (Constraint.length() == 1) { 503 switch (Constraint[0]) { 504 case 'I': // Unsigned 8-bit constant 505 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 506 if (isUInt<8>(C->getZExtValue())) 507 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 508 Op.getValueType())); 509 return; 510 511 case 'J': // Unsigned 12-bit constant 512 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 513 if (isUInt<12>(C->getZExtValue())) 514 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 515 Op.getValueType())); 516 return; 517 518 case 'K': // Signed 16-bit constant 519 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 520 if (isInt<16>(C->getSExtValue())) 521 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 522 Op.getValueType())); 523 return; 524 525 case 'L': // Signed 20-bit displacement (on all targets we support) 526 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 527 if (isInt<20>(C->getSExtValue())) 528 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 529 Op.getValueType())); 530 return; 531 532 case 'M': // 0x7fffffff 533 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 534 if (C->getZExtValue() == 0x7fffffff) 535 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 536 Op.getValueType())); 537 return; 538 } 539 } 540 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 541} 542 543//===----------------------------------------------------------------------===// 544// Calling conventions 545//===----------------------------------------------------------------------===// 546 547#include "SystemZGenCallingConv.inc" 548 549bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 550 Type *ToType) const { 551 return isTruncateFree(FromType, ToType); 552} 553 554bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 555 if (!CI->isTailCall()) 556 return false; 557 return true; 558} 559 560// Value is a value that has been passed to us in the location described by VA 561// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 562// any loads onto Chain. 563static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 564 CCValAssign &VA, SDValue Chain, 565 SDValue Value) { 566 // If the argument has been promoted from a smaller type, insert an 567 // assertion to capture this. 568 if (VA.getLocInfo() == CCValAssign::SExt) 569 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 570 DAG.getValueType(VA.getValVT())); 571 else if (VA.getLocInfo() == CCValAssign::ZExt) 572 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 573 DAG.getValueType(VA.getValVT())); 574 575 if (VA.isExtInLoc()) 576 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 577 else if (VA.getLocInfo() == CCValAssign::Indirect) 578 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 579 MachinePointerInfo(), false, false, false, 0); 580 else 581 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 582 return Value; 583} 584 585// Value is a value of type VA.getValVT() that we need to copy into 586// the location described by VA. Return a copy of Value converted to 587// VA.getValVT(). The caller is responsible for handling indirect values. 588static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 589 CCValAssign &VA, SDValue Value) { 590 switch (VA.getLocInfo()) { 591 case CCValAssign::SExt: 592 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 593 case CCValAssign::ZExt: 594 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 595 case CCValAssign::AExt: 596 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 597 case CCValAssign::Full: 598 return Value; 599 default: 600 llvm_unreachable("Unhandled getLocInfo()"); 601 } 602} 603 604SDValue SystemZTargetLowering:: 605LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 606 const SmallVectorImpl<ISD::InputArg> &Ins, 607 SDLoc DL, SelectionDAG &DAG, 608 SmallVectorImpl<SDValue> &InVals) const { 609 MachineFunction &MF = DAG.getMachineFunction(); 610 MachineFrameInfo *MFI = MF.getFrameInfo(); 611 MachineRegisterInfo &MRI = MF.getRegInfo(); 612 SystemZMachineFunctionInfo *FuncInfo = 613 MF.getInfo<SystemZMachineFunctionInfo>(); 614 const SystemZFrameLowering *TFL = 615 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 616 617 // Assign locations to all of the incoming arguments. 618 SmallVector<CCValAssign, 16> ArgLocs; 619 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 620 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 621 622 unsigned NumFixedGPRs = 0; 623 unsigned NumFixedFPRs = 0; 624 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 625 SDValue ArgValue; 626 CCValAssign &VA = ArgLocs[I]; 627 EVT LocVT = VA.getLocVT(); 628 if (VA.isRegLoc()) { 629 // Arguments passed in registers 630 const TargetRegisterClass *RC; 631 switch (LocVT.getSimpleVT().SimpleTy) { 632 default: 633 // Integers smaller than i64 should be promoted to i64. 634 llvm_unreachable("Unexpected argument type"); 635 case MVT::i32: 636 NumFixedGPRs += 1; 637 RC = &SystemZ::GR32BitRegClass; 638 break; 639 case MVT::i64: 640 NumFixedGPRs += 1; 641 RC = &SystemZ::GR64BitRegClass; 642 break; 643 case MVT::f32: 644 NumFixedFPRs += 1; 645 RC = &SystemZ::FP32BitRegClass; 646 break; 647 case MVT::f64: 648 NumFixedFPRs += 1; 649 RC = &SystemZ::FP64BitRegClass; 650 break; 651 } 652 653 unsigned VReg = MRI.createVirtualRegister(RC); 654 MRI.addLiveIn(VA.getLocReg(), VReg); 655 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 656 } else { 657 assert(VA.isMemLoc() && "Argument not register or memory"); 658 659 // Create the frame index object for this incoming parameter. 660 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 661 VA.getLocMemOffset(), true); 662 663 // Create the SelectionDAG nodes corresponding to a load 664 // from this parameter. Unpromoted ints and floats are 665 // passed as right-justified 8-byte values. 666 EVT PtrVT = getPointerTy(); 667 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 668 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 669 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 670 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 671 MachinePointerInfo::getFixedStack(FI), 672 false, false, false, 0); 673 } 674 675 // Convert the value of the argument register into the value that's 676 // being passed. 677 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 678 } 679 680 if (IsVarArg) { 681 // Save the number of non-varargs registers for later use by va_start, etc. 682 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 683 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 684 685 // Likewise the address (in the form of a frame index) of where the 686 // first stack vararg would be. The 1-byte size here is arbitrary. 687 int64_t StackSize = CCInfo.getNextStackOffset(); 688 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 689 690 // ...and a similar frame index for the caller-allocated save area 691 // that will be used to store the incoming registers. 692 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 693 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 694 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 695 696 // Store the FPR varargs in the reserved frame slots. (We store the 697 // GPRs as part of the prologue.) 698 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 699 SDValue MemOps[SystemZ::NumArgFPRs]; 700 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 701 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 702 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 703 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 704 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 705 &SystemZ::FP64BitRegClass); 706 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 707 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 708 MachinePointerInfo::getFixedStack(FI), 709 false, false, 0); 710 711 } 712 // Join the stores, which are independent of one another. 713 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 714 &MemOps[NumFixedFPRs], 715 SystemZ::NumArgFPRs - NumFixedFPRs); 716 } 717 } 718 719 return Chain; 720} 721 722static bool canUseSiblingCall(CCState ArgCCInfo, 723 SmallVectorImpl<CCValAssign> &ArgLocs) { 724 // Punt if there are any indirect or stack arguments, or if the call 725 // needs the call-saved argument register R6. 726 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 727 CCValAssign &VA = ArgLocs[I]; 728 if (VA.getLocInfo() == CCValAssign::Indirect) 729 return false; 730 if (!VA.isRegLoc()) 731 return false; 732 unsigned Reg = VA.getLocReg(); 733 if (Reg == SystemZ::R6W || Reg == SystemZ::R6D) 734 return false; 735 } 736 return true; 737} 738 739SDValue 740SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 741 SmallVectorImpl<SDValue> &InVals) const { 742 SelectionDAG &DAG = CLI.DAG; 743 SDLoc &DL = CLI.DL; 744 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 745 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 746 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 747 SDValue Chain = CLI.Chain; 748 SDValue Callee = CLI.Callee; 749 bool &IsTailCall = CLI.IsTailCall; 750 CallingConv::ID CallConv = CLI.CallConv; 751 bool IsVarArg = CLI.IsVarArg; 752 MachineFunction &MF = DAG.getMachineFunction(); 753 EVT PtrVT = getPointerTy(); 754 755 // Analyze the operands of the call, assigning locations to each operand. 756 SmallVector<CCValAssign, 16> ArgLocs; 757 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 758 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 759 760 // We don't support GuaranteedTailCallOpt, only automatically-detected 761 // sibling calls. 762 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 763 IsTailCall = false; 764 765 // Get a count of how many bytes are to be pushed on the stack. 766 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 767 768 // Mark the start of the call. 769 if (!IsTailCall) 770 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 771 DL); 772 773 // Copy argument values to their designated locations. 774 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 775 SmallVector<SDValue, 8> MemOpChains; 776 SDValue StackPtr; 777 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 778 CCValAssign &VA = ArgLocs[I]; 779 SDValue ArgValue = OutVals[I]; 780 781 if (VA.getLocInfo() == CCValAssign::Indirect) { 782 // Store the argument in a stack slot and pass its address. 783 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 784 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 785 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 786 MachinePointerInfo::getFixedStack(FI), 787 false, false, 0)); 788 ArgValue = SpillSlot; 789 } else 790 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 791 792 if (VA.isRegLoc()) 793 // Queue up the argument copies and emit them at the end. 794 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 795 else { 796 assert(VA.isMemLoc() && "Argument not register or memory"); 797 798 // Work out the address of the stack slot. Unpromoted ints and 799 // floats are passed as right-justified 8-byte values. 800 if (!StackPtr.getNode()) 801 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 802 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 803 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 804 Offset += 4; 805 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 806 DAG.getIntPtrConstant(Offset)); 807 808 // Emit the store. 809 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 810 MachinePointerInfo(), 811 false, false, 0)); 812 } 813 } 814 815 // Join the stores, which are independent of one another. 816 if (!MemOpChains.empty()) 817 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 818 &MemOpChains[0], MemOpChains.size()); 819 820 // Accept direct calls by converting symbolic call addresses to the 821 // associated Target* opcodes. Force %r1 to be used for indirect 822 // tail calls. 823 SDValue Glue; 824 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 825 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 826 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 827 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 828 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 829 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 830 } else if (IsTailCall) { 831 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 832 Glue = Chain.getValue(1); 833 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 834 } 835 836 // Build a sequence of copy-to-reg nodes, chained and glued together. 837 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 838 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 839 RegsToPass[I].second, Glue); 840 Glue = Chain.getValue(1); 841 } 842 843 // The first call operand is the chain and the second is the target address. 844 SmallVector<SDValue, 8> Ops; 845 Ops.push_back(Chain); 846 Ops.push_back(Callee); 847 848 // Add argument registers to the end of the list so that they are 849 // known live into the call. 850 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 851 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 852 RegsToPass[I].second.getValueType())); 853 854 // Glue the call to the argument copies, if any. 855 if (Glue.getNode()) 856 Ops.push_back(Glue); 857 858 // Emit the call. 859 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 860 if (IsTailCall) 861 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, &Ops[0], Ops.size()); 862 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 863 Glue = Chain.getValue(1); 864 865 // Mark the end of the call, which is glued to the call itself. 866 Chain = DAG.getCALLSEQ_END(Chain, 867 DAG.getConstant(NumBytes, PtrVT, true), 868 DAG.getConstant(0, PtrVT, true), 869 Glue, DL); 870 Glue = Chain.getValue(1); 871 872 // Assign locations to each value returned by this call. 873 SmallVector<CCValAssign, 16> RetLocs; 874 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 875 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 876 877 // Copy all of the result registers out of their specified physreg. 878 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 879 CCValAssign &VA = RetLocs[I]; 880 881 // Copy the value out, gluing the copy to the end of the call sequence. 882 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 883 VA.getLocVT(), Glue); 884 Chain = RetValue.getValue(1); 885 Glue = RetValue.getValue(2); 886 887 // Convert the value of the return register into the value that's 888 // being returned. 889 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 890 } 891 892 return Chain; 893} 894 895SDValue 896SystemZTargetLowering::LowerReturn(SDValue Chain, 897 CallingConv::ID CallConv, bool IsVarArg, 898 const SmallVectorImpl<ISD::OutputArg> &Outs, 899 const SmallVectorImpl<SDValue> &OutVals, 900 SDLoc DL, SelectionDAG &DAG) const { 901 MachineFunction &MF = DAG.getMachineFunction(); 902 903 // Assign locations to each returned value. 904 SmallVector<CCValAssign, 16> RetLocs; 905 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 906 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 907 908 // Quick exit for void returns 909 if (RetLocs.empty()) 910 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 911 912 // Copy the result values into the output registers. 913 SDValue Glue; 914 SmallVector<SDValue, 4> RetOps; 915 RetOps.push_back(Chain); 916 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 917 CCValAssign &VA = RetLocs[I]; 918 SDValue RetValue = OutVals[I]; 919 920 // Make the return register live on exit. 921 assert(VA.isRegLoc() && "Can only return in registers!"); 922 923 // Promote the value as required. 924 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 925 926 // Chain and glue the copies together. 927 unsigned Reg = VA.getLocReg(); 928 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 929 Glue = Chain.getValue(1); 930 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 931 } 932 933 // Update chain and glue. 934 RetOps[0] = Chain; 935 if (Glue.getNode()) 936 RetOps.push_back(Glue); 937 938 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 939 RetOps.data(), RetOps.size()); 940} 941 942// CC is a comparison that will be implemented using an integer or 943// floating-point comparison. Return the condition code mask for 944// a branch on true. In the integer case, CCMASK_CMP_UO is set for 945// unsigned comparisons and clear for signed ones. In the floating-point 946// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 947static unsigned CCMaskForCondCode(ISD::CondCode CC) { 948#define CONV(X) \ 949 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 950 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 951 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 952 953 switch (CC) { 954 default: 955 llvm_unreachable("Invalid integer condition!"); 956 957 CONV(EQ); 958 CONV(NE); 959 CONV(GT); 960 CONV(GE); 961 CONV(LT); 962 CONV(LE); 963 964 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 965 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 966 } 967#undef CONV 968} 969 970// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 971// can be converted to a comparison against zero, adjust the operands 972// as necessary. 973static void adjustZeroCmp(SelectionDAG &DAG, bool &IsUnsigned, 974 SDValue &CmpOp0, SDValue &CmpOp1, 975 unsigned &CCMask) { 976 if (IsUnsigned) 977 return; 978 979 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(CmpOp1.getNode()); 980 if (!ConstOp1) 981 return; 982 983 int64_t Value = ConstOp1->getSExtValue(); 984 if ((Value == -1 && CCMask == SystemZ::CCMASK_CMP_GT) || 985 (Value == -1 && CCMask == SystemZ::CCMASK_CMP_LE) || 986 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_LT) || 987 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_GE)) { 988 CCMask ^= SystemZ::CCMASK_CMP_EQ; 989 CmpOp1 = DAG.getConstant(0, CmpOp1.getValueType()); 990 } 991} 992 993// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 994// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 995static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 996 SDValue &CmpOp0, SDValue &CmpOp1, 997 unsigned &CCMask) { 998 // For us to make any changes, it must a comparison between a single-use 999 // load and a constant. 1000 if (!CmpOp0.hasOneUse() || 1001 CmpOp0.getOpcode() != ISD::LOAD || 1002 CmpOp1.getOpcode() != ISD::Constant) 1003 return; 1004 1005 // We must have an 8- or 16-bit load. 1006 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 1007 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1008 if (NumBits != 8 && NumBits != 16) 1009 return; 1010 1011 // The load must be an extending one and the constant must be within the 1012 // range of the unextended value. 1013 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 1014 uint64_t Value = Constant->getZExtValue(); 1015 uint64_t Mask = (1 << NumBits) - 1; 1016 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1017 int64_t SignedValue = Constant->getSExtValue(); 1018 if (uint64_t(SignedValue) + (1ULL << (NumBits - 1)) > Mask) 1019 return; 1020 // Unsigned comparison between two sign-extended values is equivalent 1021 // to unsigned comparison between two zero-extended values. 1022 if (IsUnsigned) 1023 Value &= Mask; 1024 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 1025 CCMask == SystemZ::CCMASK_CMP_NE) 1026 // Any choice of IsUnsigned is OK for equality comparisons. 1027 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 1028 // but since we use CLHHSI for zero extensions, it seems better 1029 // to be consistent and do the same here. 1030 Value &= Mask, IsUnsigned = true; 1031 else if (NumBits == 8) { 1032 // Try to treat the comparison as unsigned, so that we can use CLI. 1033 // Adjust CCMask and Value as necessary. 1034 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 1035 // Test whether the high bit of the byte is set. 1036 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 1037 else if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_GE) 1038 // Test whether the high bit of the byte is clear. 1039 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 1040 else 1041 // No instruction exists for this combination. 1042 return; 1043 } 1044 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1045 if (Value > Mask) 1046 return; 1047 // Signed comparison between two zero-extended values is equivalent 1048 // to unsigned comparison. 1049 IsUnsigned = true; 1050 } else 1051 return; 1052 1053 // Make sure that the first operand is an i32 of the right extension type. 1054 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 1055 if (CmpOp0.getValueType() != MVT::i32 || 1056 Load->getExtensionType() != ExtType) 1057 CmpOp0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1058 Load->getChain(), Load->getBasePtr(), 1059 Load->getPointerInfo(), Load->getMemoryVT(), 1060 Load->isVolatile(), Load->isNonTemporal(), 1061 Load->getAlignment()); 1062 1063 // Make sure that the second operand is an i32 with the right value. 1064 if (CmpOp1.getValueType() != MVT::i32 || 1065 Value != Constant->getZExtValue()) 1066 CmpOp1 = DAG.getConstant(Value, MVT::i32); 1067} 1068 1069// Return true if a comparison described by CCMask, CmpOp0 and CmpOp1 1070// is an equality comparison that is better implemented using unsigned 1071// rather than signed comparison instructions. 1072static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0, 1073 SDValue CmpOp1, unsigned CCMask) { 1074 // The test must be for equality or inequality. 1075 if (CCMask != SystemZ::CCMASK_CMP_EQ && CCMask != SystemZ::CCMASK_CMP_NE) 1076 return false; 1077 1078 if (CmpOp1.getOpcode() == ISD::Constant) { 1079 uint64_t Value = cast<ConstantSDNode>(CmpOp1)->getSExtValue(); 1080 1081 // If we're comparing with memory, prefer unsigned comparisons for 1082 // values that are in the unsigned 16-bit range but not the signed 1083 // 16-bit range. We want to use CLFHSI and CLGHSI. 1084 if (CmpOp0.hasOneUse() && 1085 ISD::isNormalLoad(CmpOp0.getNode()) && 1086 (Value >= 32768 && Value < 65536)) 1087 return true; 1088 1089 // Use unsigned comparisons for values that are in the CLGFI range 1090 // but not in the CGFI range. 1091 if (CmpOp0.getValueType() == MVT::i64 && (Value >> 31) == 1) 1092 return true; 1093 1094 return false; 1095 } 1096 1097 // Prefer CL for zero-extended loads. 1098 if (CmpOp1.getOpcode() == ISD::ZERO_EXTEND || 1099 ISD::isZEXTLoad(CmpOp1.getNode())) 1100 return true; 1101 1102 // ...and for "in-register" zero extensions. 1103 if (CmpOp1.getOpcode() == ISD::AND && CmpOp1.getValueType() == MVT::i64) { 1104 SDValue Mask = CmpOp1.getOperand(1); 1105 if (Mask.getOpcode() == ISD::Constant && 1106 cast<ConstantSDNode>(Mask)->getZExtValue() == 0xffffffff) 1107 return true; 1108 } 1109 1110 return false; 1111} 1112 1113// Return true if Op is either an unextended load, or a load with the 1114// extension type given by IsUnsigned. 1115static bool isNaturalMemoryOperand(SDValue Op, bool IsUnsigned) { 1116 LoadSDNode *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1117 if (Load) 1118 switch (Load->getExtensionType()) { 1119 case ISD::NON_EXTLOAD: 1120 case ISD::EXTLOAD: 1121 return true; 1122 case ISD::SEXTLOAD: 1123 return !IsUnsigned; 1124 case ISD::ZEXTLOAD: 1125 return IsUnsigned; 1126 default: 1127 break; 1128 } 1129 return false; 1130} 1131 1132// Return true if it is better to swap comparison operands Op0 and Op1. 1133// IsUnsigned says whether an integer comparison is signed or unsigned. 1134static bool shouldSwapCmpOperands(SDValue Op0, SDValue Op1, 1135 bool IsUnsigned) { 1136 // Leave f128 comparisons alone, since they have no memory forms. 1137 if (Op0.getValueType() == MVT::f128) 1138 return false; 1139 1140 // Always keep a floating-point constant second, since comparisons with 1141 // zero can use LOAD TEST and comparisons with other constants make a 1142 // natural memory operand. 1143 if (isa<ConstantFPSDNode>(Op1)) 1144 return false; 1145 1146 // Never swap comparisons with zero since there are many ways to optimize 1147 // those later. 1148 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1); 1149 if (COp1 && COp1->getZExtValue() == 0) 1150 return false; 1151 1152 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1153 // In that case we generally prefer the memory to be second. 1154 if ((isNaturalMemoryOperand(Op0, IsUnsigned) && Op0.hasOneUse()) && 1155 !(isNaturalMemoryOperand(Op1, IsUnsigned) && Op1.hasOneUse())) { 1156 // The only exceptions are when the second operand is a constant and 1157 // we can use things like CHHSI. 1158 if (!COp1) 1159 return true; 1160 if (IsUnsigned) { 1161 // The memory-immediate instructions require 16-bit unsigned integers. 1162 if (isUInt<16>(COp1->getZExtValue())) 1163 return false; 1164 } else { 1165 // There are no comparisons between integers and signed memory bytes. 1166 // The others require 16-bit signed integers. 1167 if (cast<LoadSDNode>(Op0.getNode())->getMemoryVT() == MVT::i8 || 1168 isInt<16>(COp1->getSExtValue())) 1169 return false; 1170 } 1171 return true; 1172 } 1173 return false; 1174} 1175 1176// Return a target node that compares CmpOp0 with CmpOp1 and stores a 1177// 2-bit result in CC. Set CCValid to the CCMASK_* of all possible 1178// 2-bit results and CCMask to the subset of those results that are 1179// associated with Cond. 1180static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 1181 ISD::CondCode Cond, unsigned &CCValid, 1182 unsigned &CCMask) { 1183 bool IsUnsigned = false; 1184 CCMask = CCMaskForCondCode(Cond); 1185 if (CmpOp0.getValueType().isFloatingPoint()) 1186 CCValid = SystemZ::CCMASK_FCMP; 1187 else { 1188 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 1189 CCValid = SystemZ::CCMASK_ICMP; 1190 CCMask &= CCValid; 1191 adjustZeroCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1192 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1193 if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask)) 1194 IsUnsigned = true; 1195 } 1196 1197 if (shouldSwapCmpOperands(CmpOp0, CmpOp1, IsUnsigned)) { 1198 std::swap(CmpOp0, CmpOp1); 1199 CCMask = ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1200 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1201 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1202 (CCMask & SystemZ::CCMASK_CMP_UO)); 1203 } 1204 1205 SDLoc DL(CmpOp0); 1206 return DAG.getNode((IsUnsigned ? SystemZISD::UCMP : SystemZISD::CMP), 1207 DL, MVT::Glue, CmpOp0, CmpOp1); 1208} 1209 1210// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1211// 64 bits. Extend is the extension type to use. Store the high part 1212// in Hi and the low part in Lo. 1213static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1214 unsigned Extend, SDValue Op0, SDValue Op1, 1215 SDValue &Hi, SDValue &Lo) { 1216 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1217 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1218 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1219 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1220 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1221 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1222} 1223 1224// Lower a binary operation that produces two VT results, one in each 1225// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1226// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1227// on the extended Op0 and (unextended) Op1. Store the even register result 1228// in Even and the odd register result in Odd. 1229static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1230 unsigned Extend, unsigned Opcode, 1231 SDValue Op0, SDValue Op1, 1232 SDValue &Even, SDValue &Odd) { 1233 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1234 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1235 SDValue(In128, 0), Op1); 1236 bool Is32Bit = is32Bit(VT); 1237 SDValue SubReg0 = DAG.getTargetConstant(SystemZ::even128(Is32Bit), VT); 1238 SDValue SubReg1 = DAG.getTargetConstant(SystemZ::odd128(Is32Bit), VT); 1239 SDNode *Reg0 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1240 VT, Result, SubReg0); 1241 SDNode *Reg1 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1242 VT, Result, SubReg1); 1243 Even = SDValue(Reg0, 0); 1244 Odd = SDValue(Reg1, 0); 1245} 1246 1247SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1248 SDValue Chain = Op.getOperand(0); 1249 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1250 SDValue CmpOp0 = Op.getOperand(2); 1251 SDValue CmpOp1 = Op.getOperand(3); 1252 SDValue Dest = Op.getOperand(4); 1253 SDLoc DL(Op); 1254 1255 unsigned CCValid, CCMask; 1256 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1257 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1258 Chain, DAG.getConstant(CCValid, MVT::i32), 1259 DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 1260} 1261 1262SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1263 SelectionDAG &DAG) const { 1264 SDValue CmpOp0 = Op.getOperand(0); 1265 SDValue CmpOp1 = Op.getOperand(1); 1266 SDValue TrueOp = Op.getOperand(2); 1267 SDValue FalseOp = Op.getOperand(3); 1268 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1269 SDLoc DL(Op); 1270 1271 unsigned CCValid, CCMask; 1272 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1273 1274 SmallVector<SDValue, 5> Ops; 1275 Ops.push_back(TrueOp); 1276 Ops.push_back(FalseOp); 1277 Ops.push_back(DAG.getConstant(CCValid, MVT::i32)); 1278 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1279 Ops.push_back(Flags); 1280 1281 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1282 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1283} 1284 1285SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1286 SelectionDAG &DAG) const { 1287 SDLoc DL(Node); 1288 const GlobalValue *GV = Node->getGlobal(); 1289 int64_t Offset = Node->getOffset(); 1290 EVT PtrVT = getPointerTy(); 1291 Reloc::Model RM = TM.getRelocationModel(); 1292 CodeModel::Model CM = TM.getCodeModel(); 1293 1294 SDValue Result; 1295 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1296 // Make sure that the offset is aligned to a halfword. If it isn't, 1297 // create an "anchor" at the previous 12-bit boundary. 1298 // FIXME check whether there is a better way of handling this. 1299 if (Offset & 1) { 1300 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 1301 Offset & ~uint64_t(0xfff)); 1302 Offset &= 0xfff; 1303 } else { 1304 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Offset); 1305 Offset = 0; 1306 } 1307 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1308 } else { 1309 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1310 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1311 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1312 MachinePointerInfo::getGOT(), false, false, false, 0); 1313 } 1314 1315 // If there was a non-zero offset that we didn't fold, create an explicit 1316 // addition for it. 1317 if (Offset != 0) 1318 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1319 DAG.getConstant(Offset, PtrVT)); 1320 1321 return Result; 1322} 1323 1324SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1325 SelectionDAG &DAG) const { 1326 SDLoc DL(Node); 1327 const GlobalValue *GV = Node->getGlobal(); 1328 EVT PtrVT = getPointerTy(); 1329 TLSModel::Model model = TM.getTLSModel(GV); 1330 1331 if (model != TLSModel::LocalExec) 1332 llvm_unreachable("only local-exec TLS mode supported"); 1333 1334 // The high part of the thread pointer is in access register 0. 1335 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1336 DAG.getConstant(0, MVT::i32)); 1337 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1338 1339 // The low part of the thread pointer is in access register 1. 1340 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1341 DAG.getConstant(1, MVT::i32)); 1342 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1343 1344 // Merge them into a single 64-bit address. 1345 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1346 DAG.getConstant(32, PtrVT)); 1347 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1348 1349 // Get the offset of GA from the thread pointer. 1350 SystemZConstantPoolValue *CPV = 1351 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1352 1353 // Force the offset into the constant pool and load it from there. 1354 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1355 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1356 CPAddr, MachinePointerInfo::getConstantPool(), 1357 false, false, false, 0); 1358 1359 // Add the base and offset together. 1360 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1361} 1362 1363SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1364 SelectionDAG &DAG) const { 1365 SDLoc DL(Node); 1366 const BlockAddress *BA = Node->getBlockAddress(); 1367 int64_t Offset = Node->getOffset(); 1368 EVT PtrVT = getPointerTy(); 1369 1370 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1371 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1372 return Result; 1373} 1374 1375SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1376 SelectionDAG &DAG) const { 1377 SDLoc DL(JT); 1378 EVT PtrVT = getPointerTy(); 1379 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1380 1381 // Use LARL to load the address of the table. 1382 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1383} 1384 1385SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1386 SelectionDAG &DAG) const { 1387 SDLoc DL(CP); 1388 EVT PtrVT = getPointerTy(); 1389 1390 SDValue Result; 1391 if (CP->isMachineConstantPoolEntry()) 1392 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1393 CP->getAlignment()); 1394 else 1395 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1396 CP->getAlignment(), CP->getOffset()); 1397 1398 // Use LARL to load the address of the constant pool entry. 1399 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1400} 1401 1402SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1403 SelectionDAG &DAG) const { 1404 SDLoc DL(Op); 1405 SDValue In = Op.getOperand(0); 1406 EVT InVT = In.getValueType(); 1407 EVT ResVT = Op.getValueType(); 1408 1409 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1410 SDValue Shift32 = DAG.getConstant(32, MVT::i64); 1411 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1412 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1413 SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); 1414 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); 1415 SDNode *Out = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1416 MVT::f32, Out64, SubReg32); 1417 return SDValue(Out, 0); 1418 } 1419 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1420 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1421 SDNode *In64 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1422 MVT::f64, SDValue(U64, 0), In, SubReg32); 1423 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, SDValue(In64, 0)); 1424 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); 1425 SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1426 return Out; 1427 } 1428 llvm_unreachable("Unexpected bitcast combination"); 1429} 1430 1431SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1432 SelectionDAG &DAG) const { 1433 MachineFunction &MF = DAG.getMachineFunction(); 1434 SystemZMachineFunctionInfo *FuncInfo = 1435 MF.getInfo<SystemZMachineFunctionInfo>(); 1436 EVT PtrVT = getPointerTy(); 1437 1438 SDValue Chain = Op.getOperand(0); 1439 SDValue Addr = Op.getOperand(1); 1440 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1441 SDLoc DL(Op); 1442 1443 // The initial values of each field. 1444 const unsigned NumFields = 4; 1445 SDValue Fields[NumFields] = { 1446 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1447 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1448 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1449 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1450 }; 1451 1452 // Store each field into its respective slot. 1453 SDValue MemOps[NumFields]; 1454 unsigned Offset = 0; 1455 for (unsigned I = 0; I < NumFields; ++I) { 1456 SDValue FieldAddr = Addr; 1457 if (Offset != 0) 1458 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1459 DAG.getIntPtrConstant(Offset)); 1460 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1461 MachinePointerInfo(SV, Offset), 1462 false, false, 0); 1463 Offset += 8; 1464 } 1465 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1466} 1467 1468SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1469 SelectionDAG &DAG) const { 1470 SDValue Chain = Op.getOperand(0); 1471 SDValue DstPtr = Op.getOperand(1); 1472 SDValue SrcPtr = Op.getOperand(2); 1473 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1474 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1475 SDLoc DL(Op); 1476 1477 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1478 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1479 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1480} 1481 1482SDValue SystemZTargetLowering:: 1483lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1484 SDValue Chain = Op.getOperand(0); 1485 SDValue Size = Op.getOperand(1); 1486 SDLoc DL(Op); 1487 1488 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1489 1490 // Get a reference to the stack pointer. 1491 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1492 1493 // Get the new stack pointer value. 1494 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1495 1496 // Copy the new stack pointer back. 1497 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1498 1499 // The allocated data lives above the 160 bytes allocated for the standard 1500 // frame, plus any outgoing stack arguments. We don't know how much that 1501 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1502 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1503 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1504 1505 SDValue Ops[2] = { Result, Chain }; 1506 return DAG.getMergeValues(Ops, 2, DL); 1507} 1508 1509SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 1510 SelectionDAG &DAG) const { 1511 EVT VT = Op.getValueType(); 1512 SDLoc DL(Op); 1513 SDValue Ops[2]; 1514 if (is32Bit(VT)) 1515 // Just do a normal 64-bit multiplication and extract the results. 1516 // We define this so that it can be used for constant division. 1517 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 1518 Op.getOperand(1), Ops[1], Ops[0]); 1519 else { 1520 // Do a full 128-bit multiplication based on UMUL_LOHI64: 1521 // 1522 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 1523 // 1524 // but using the fact that the upper halves are either all zeros 1525 // or all ones: 1526 // 1527 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 1528 // 1529 // and grouping the right terms together since they are quicker than the 1530 // multiplication: 1531 // 1532 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 1533 SDValue C63 = DAG.getConstant(63, MVT::i64); 1534 SDValue LL = Op.getOperand(0); 1535 SDValue RL = Op.getOperand(1); 1536 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 1537 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 1538 // UMUL_LOHI64 returns the low result in the odd register and the high 1539 // result in the even register. SMUL_LOHI is defined to return the 1540 // low half first, so the results are in reverse order. 1541 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1542 LL, RL, Ops[1], Ops[0]); 1543 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 1544 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 1545 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 1546 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 1547 } 1548 return DAG.getMergeValues(Ops, 2, DL); 1549} 1550 1551SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1552 SelectionDAG &DAG) const { 1553 EVT VT = Op.getValueType(); 1554 SDLoc DL(Op); 1555 SDValue Ops[2]; 1556 if (is32Bit(VT)) 1557 // Just do a normal 64-bit multiplication and extract the results. 1558 // We define this so that it can be used for constant division. 1559 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 1560 Op.getOperand(1), Ops[1], Ops[0]); 1561 else 1562 // UMUL_LOHI64 returns the low result in the odd register and the high 1563 // result in the even register. UMUL_LOHI is defined to return the 1564 // low half first, so the results are in reverse order. 1565 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1566 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1567 return DAG.getMergeValues(Ops, 2, DL); 1568} 1569 1570SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1571 SelectionDAG &DAG) const { 1572 SDValue Op0 = Op.getOperand(0); 1573 SDValue Op1 = Op.getOperand(1); 1574 EVT VT = Op.getValueType(); 1575 SDLoc DL(Op); 1576 unsigned Opcode; 1577 1578 // We use DSGF for 32-bit division. 1579 if (is32Bit(VT)) { 1580 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1581 Opcode = SystemZISD::SDIVREM32; 1582 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 1583 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 1584 Opcode = SystemZISD::SDIVREM32; 1585 } else 1586 Opcode = SystemZISD::SDIVREM64; 1587 1588 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1589 // input is "don't care". The instruction returns the remainder in 1590 // the even register and the quotient in the odd register. 1591 SDValue Ops[2]; 1592 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 1593 Op0, Op1, Ops[1], Ops[0]); 1594 return DAG.getMergeValues(Ops, 2, DL); 1595} 1596 1597SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1598 SelectionDAG &DAG) const { 1599 EVT VT = Op.getValueType(); 1600 SDLoc DL(Op); 1601 1602 // DL(G) uses a double-width dividend, so we need to clear the even 1603 // register in the GR128 input. The instruction returns the remainder 1604 // in the even register and the quotient in the odd register. 1605 SDValue Ops[2]; 1606 if (is32Bit(VT)) 1607 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1608 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1609 else 1610 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1611 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1612 return DAG.getMergeValues(Ops, 2, DL); 1613} 1614 1615SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1616 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1617 1618 // Get the known-zero masks for each operand. 1619 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1620 APInt KnownZero[2], KnownOne[2]; 1621 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1622 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1623 1624 // See if the upper 32 bits of one operand and the lower 32 bits of the 1625 // other are known zero. They are the low and high operands respectively. 1626 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1627 KnownZero[1].getZExtValue() }; 1628 unsigned High, Low; 1629 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1630 High = 1, Low = 0; 1631 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1632 High = 0, Low = 1; 1633 else 1634 return Op; 1635 1636 SDValue LowOp = Ops[Low]; 1637 SDValue HighOp = Ops[High]; 1638 1639 // If the high part is a constant, we're better off using IILH. 1640 if (HighOp.getOpcode() == ISD::Constant) 1641 return Op; 1642 1643 // If the low part is a constant that is outside the range of LHI, 1644 // then we're better off using IILF. 1645 if (LowOp.getOpcode() == ISD::Constant) { 1646 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1647 if (!isInt<16>(Value)) 1648 return Op; 1649 } 1650 1651 // Check whether the high part is an AND that doesn't change the 1652 // high 32 bits and just masks out low bits. We can skip it if so. 1653 if (HighOp.getOpcode() == ISD::AND && 1654 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1655 ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); 1656 uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; 1657 if ((Mask >> 32) == 0xffffffff) 1658 HighOp = HighOp.getOperand(0); 1659 } 1660 1661 // Take advantage of the fact that all GR32 operations only change the 1662 // low 32 bits by truncating Low to an i32 and inserting it directly 1663 // using a subreg. The interesting cases are those where the truncation 1664 // can be folded. 1665 SDLoc DL(Op); 1666 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1667 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1668 SDNode *Result = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1669 MVT::i64, HighOp, Low32, SubReg32); 1670 return SDValue(Result, 0); 1671} 1672 1673// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1674// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1675SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1676 SelectionDAG &DAG, 1677 unsigned Opcode) const { 1678 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1679 1680 // 32-bit operations need no code outside the main loop. 1681 EVT NarrowVT = Node->getMemoryVT(); 1682 EVT WideVT = MVT::i32; 1683 if (NarrowVT == WideVT) 1684 return Op; 1685 1686 int64_t BitSize = NarrowVT.getSizeInBits(); 1687 SDValue ChainIn = Node->getChain(); 1688 SDValue Addr = Node->getBasePtr(); 1689 SDValue Src2 = Node->getVal(); 1690 MachineMemOperand *MMO = Node->getMemOperand(); 1691 SDLoc DL(Node); 1692 EVT PtrVT = Addr.getValueType(); 1693 1694 // Convert atomic subtracts of constants into additions. 1695 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1696 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1697 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1698 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1699 } 1700 1701 // Get the address of the containing word. 1702 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1703 DAG.getConstant(-4, PtrVT)); 1704 1705 // Get the number of bits that the word must be rotated left in order 1706 // to bring the field to the top bits of a GR32. 1707 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1708 DAG.getConstant(3, PtrVT)); 1709 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1710 1711 // Get the complementing shift amount, for rotating a field in the top 1712 // bits back to its proper position. 1713 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1714 DAG.getConstant(0, WideVT), BitShift); 1715 1716 // Extend the source operand to 32 bits and prepare it for the inner loop. 1717 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1718 // operations require the source to be shifted in advance. (This shift 1719 // can be folded if the source is constant.) For AND and NAND, the lower 1720 // bits must be set, while for other opcodes they should be left clear. 1721 if (Opcode != SystemZISD::ATOMIC_SWAPW) 1722 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 1723 DAG.getConstant(32 - BitSize, WideVT)); 1724 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 1725 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 1726 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 1727 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 1728 1729 // Construct the ATOMIC_LOADW_* node. 1730 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1731 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 1732 DAG.getConstant(BitSize, WideVT) }; 1733 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 1734 array_lengthof(Ops), 1735 NarrowVT, MMO); 1736 1737 // Rotate the result of the final CS so that the field is in the lower 1738 // bits of a GR32, then truncate it. 1739 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 1740 DAG.getConstant(BitSize, WideVT)); 1741 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 1742 1743 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 1744 return DAG.getMergeValues(RetOps, 2, DL); 1745} 1746 1747// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 1748// into a fullword ATOMIC_CMP_SWAPW operation. 1749SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 1750 SelectionDAG &DAG) const { 1751 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1752 1753 // We have native support for 32-bit compare and swap. 1754 EVT NarrowVT = Node->getMemoryVT(); 1755 EVT WideVT = MVT::i32; 1756 if (NarrowVT == WideVT) 1757 return Op; 1758 1759 int64_t BitSize = NarrowVT.getSizeInBits(); 1760 SDValue ChainIn = Node->getOperand(0); 1761 SDValue Addr = Node->getOperand(1); 1762 SDValue CmpVal = Node->getOperand(2); 1763 SDValue SwapVal = Node->getOperand(3); 1764 MachineMemOperand *MMO = Node->getMemOperand(); 1765 SDLoc DL(Node); 1766 EVT PtrVT = Addr.getValueType(); 1767 1768 // Get the address of the containing word. 1769 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1770 DAG.getConstant(-4, PtrVT)); 1771 1772 // Get the number of bits that the word must be rotated left in order 1773 // to bring the field to the top bits of a GR32. 1774 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1775 DAG.getConstant(3, PtrVT)); 1776 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1777 1778 // Get the complementing shift amount, for rotating a field in the top 1779 // bits back to its proper position. 1780 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1781 DAG.getConstant(0, WideVT), BitShift); 1782 1783 // Construct the ATOMIC_CMP_SWAPW node. 1784 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1785 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 1786 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 1787 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 1788 VTList, Ops, array_lengthof(Ops), 1789 NarrowVT, MMO); 1790 return AtomicOp; 1791} 1792 1793SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 1794 SelectionDAG &DAG) const { 1795 MachineFunction &MF = DAG.getMachineFunction(); 1796 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1797 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 1798 SystemZ::R15D, Op.getValueType()); 1799} 1800 1801SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 1802 SelectionDAG &DAG) const { 1803 MachineFunction &MF = DAG.getMachineFunction(); 1804 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1805 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 1806 SystemZ::R15D, Op.getOperand(1)); 1807} 1808 1809SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 1810 SelectionDAG &DAG) const { 1811 switch (Op.getOpcode()) { 1812 case ISD::BR_CC: 1813 return lowerBR_CC(Op, DAG); 1814 case ISD::SELECT_CC: 1815 return lowerSELECT_CC(Op, DAG); 1816 case ISD::GlobalAddress: 1817 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 1818 case ISD::GlobalTLSAddress: 1819 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 1820 case ISD::BlockAddress: 1821 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 1822 case ISD::JumpTable: 1823 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 1824 case ISD::ConstantPool: 1825 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 1826 case ISD::BITCAST: 1827 return lowerBITCAST(Op, DAG); 1828 case ISD::VASTART: 1829 return lowerVASTART(Op, DAG); 1830 case ISD::VACOPY: 1831 return lowerVACOPY(Op, DAG); 1832 case ISD::DYNAMIC_STACKALLOC: 1833 return lowerDYNAMIC_STACKALLOC(Op, DAG); 1834 case ISD::SMUL_LOHI: 1835 return lowerSMUL_LOHI(Op, DAG); 1836 case ISD::UMUL_LOHI: 1837 return lowerUMUL_LOHI(Op, DAG); 1838 case ISD::SDIVREM: 1839 return lowerSDIVREM(Op, DAG); 1840 case ISD::UDIVREM: 1841 return lowerUDIVREM(Op, DAG); 1842 case ISD::OR: 1843 return lowerOR(Op, DAG); 1844 case ISD::ATOMIC_SWAP: 1845 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 1846 case ISD::ATOMIC_LOAD_ADD: 1847 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 1848 case ISD::ATOMIC_LOAD_SUB: 1849 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 1850 case ISD::ATOMIC_LOAD_AND: 1851 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 1852 case ISD::ATOMIC_LOAD_OR: 1853 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 1854 case ISD::ATOMIC_LOAD_XOR: 1855 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 1856 case ISD::ATOMIC_LOAD_NAND: 1857 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 1858 case ISD::ATOMIC_LOAD_MIN: 1859 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 1860 case ISD::ATOMIC_LOAD_MAX: 1861 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 1862 case ISD::ATOMIC_LOAD_UMIN: 1863 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 1864 case ISD::ATOMIC_LOAD_UMAX: 1865 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 1866 case ISD::ATOMIC_CMP_SWAP: 1867 return lowerATOMIC_CMP_SWAP(Op, DAG); 1868 case ISD::STACKSAVE: 1869 return lowerSTACKSAVE(Op, DAG); 1870 case ISD::STACKRESTORE: 1871 return lowerSTACKRESTORE(Op, DAG); 1872 default: 1873 llvm_unreachable("Unexpected node to lower"); 1874 } 1875} 1876 1877const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 1878#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 1879 switch (Opcode) { 1880 OPCODE(RET_FLAG); 1881 OPCODE(CALL); 1882 OPCODE(SIBCALL); 1883 OPCODE(PCREL_WRAPPER); 1884 OPCODE(CMP); 1885 OPCODE(UCMP); 1886 OPCODE(BR_CCMASK); 1887 OPCODE(SELECT_CCMASK); 1888 OPCODE(ADJDYNALLOC); 1889 OPCODE(EXTRACT_ACCESS); 1890 OPCODE(UMUL_LOHI64); 1891 OPCODE(SDIVREM64); 1892 OPCODE(UDIVREM32); 1893 OPCODE(UDIVREM64); 1894 OPCODE(MVC); 1895 OPCODE(CLC); 1896 OPCODE(STRCMP); 1897 OPCODE(STPCPY); 1898 OPCODE(SEARCH_STRING); 1899 OPCODE(IPM); 1900 OPCODE(ATOMIC_SWAPW); 1901 OPCODE(ATOMIC_LOADW_ADD); 1902 OPCODE(ATOMIC_LOADW_SUB); 1903 OPCODE(ATOMIC_LOADW_AND); 1904 OPCODE(ATOMIC_LOADW_OR); 1905 OPCODE(ATOMIC_LOADW_XOR); 1906 OPCODE(ATOMIC_LOADW_NAND); 1907 OPCODE(ATOMIC_LOADW_MIN); 1908 OPCODE(ATOMIC_LOADW_MAX); 1909 OPCODE(ATOMIC_LOADW_UMIN); 1910 OPCODE(ATOMIC_LOADW_UMAX); 1911 OPCODE(ATOMIC_CMP_SWAPW); 1912 } 1913 return NULL; 1914#undef OPCODE 1915} 1916 1917//===----------------------------------------------------------------------===// 1918// Custom insertion 1919//===----------------------------------------------------------------------===// 1920 1921// Create a new basic block after MBB. 1922static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 1923 MachineFunction &MF = *MBB->getParent(); 1924 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1925 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 1926 return NewMBB; 1927} 1928 1929// Split MBB after MI and return the new block (the one that contains 1930// instructions after MI). 1931static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 1932 MachineBasicBlock *MBB) { 1933 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1934 NewMBB->splice(NewMBB->begin(), MBB, 1935 llvm::next(MachineBasicBlock::iterator(MI)), 1936 MBB->end()); 1937 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1938 return NewMBB; 1939} 1940 1941// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 1942MachineBasicBlock * 1943SystemZTargetLowering::emitSelect(MachineInstr *MI, 1944 MachineBasicBlock *MBB) const { 1945 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1946 1947 unsigned DestReg = MI->getOperand(0).getReg(); 1948 unsigned TrueReg = MI->getOperand(1).getReg(); 1949 unsigned FalseReg = MI->getOperand(2).getReg(); 1950 unsigned CCValid = MI->getOperand(3).getImm(); 1951 unsigned CCMask = MI->getOperand(4).getImm(); 1952 DebugLoc DL = MI->getDebugLoc(); 1953 1954 MachineBasicBlock *StartMBB = MBB; 1955 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 1956 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 1957 1958 // StartMBB: 1959 // BRC CCMask, JoinMBB 1960 // # fallthrough to FalseMBB 1961 MBB = StartMBB; 1962 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 1963 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 1964 MBB->addSuccessor(JoinMBB); 1965 MBB->addSuccessor(FalseMBB); 1966 1967 // FalseMBB: 1968 // # fallthrough to JoinMBB 1969 MBB = FalseMBB; 1970 MBB->addSuccessor(JoinMBB); 1971 1972 // JoinMBB: 1973 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 1974 // ... 1975 MBB = JoinMBB; 1976 BuildMI(*MBB, MBB->begin(), DL, TII->get(SystemZ::PHI), DestReg) 1977 .addReg(TrueReg).addMBB(StartMBB) 1978 .addReg(FalseReg).addMBB(FalseMBB); 1979 1980 MI->eraseFromParent(); 1981 return JoinMBB; 1982} 1983 1984// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 1985// StoreOpcode is the store to use and Invert says whether the store should 1986// happen when the condition is false rather than true. If a STORE ON 1987// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 1988MachineBasicBlock * 1989SystemZTargetLowering::emitCondStore(MachineInstr *MI, 1990 MachineBasicBlock *MBB, 1991 unsigned StoreOpcode, unsigned STOCOpcode, 1992 bool Invert) const { 1993 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1994 1995 unsigned SrcReg = MI->getOperand(0).getReg(); 1996 MachineOperand Base = MI->getOperand(1); 1997 int64_t Disp = MI->getOperand(2).getImm(); 1998 unsigned IndexReg = MI->getOperand(3).getReg(); 1999 unsigned CCValid = MI->getOperand(4).getImm(); 2000 unsigned CCMask = MI->getOperand(5).getImm(); 2001 DebugLoc DL = MI->getDebugLoc(); 2002 2003 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 2004 2005 // Use STOCOpcode if possible. We could use different store patterns in 2006 // order to avoid matching the index register, but the performance trade-offs 2007 // might be more complicated in that case. 2008 if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) { 2009 if (Invert) 2010 CCMask ^= CCValid; 2011 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 2012 .addReg(SrcReg).addOperand(Base).addImm(Disp) 2013 .addImm(CCValid).addImm(CCMask); 2014 MI->eraseFromParent(); 2015 return MBB; 2016 } 2017 2018 // Get the condition needed to branch around the store. 2019 if (!Invert) 2020 CCMask ^= CCValid; 2021 2022 MachineBasicBlock *StartMBB = MBB; 2023 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 2024 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2025 2026 // StartMBB: 2027 // BRC CCMask, JoinMBB 2028 // # fallthrough to FalseMBB 2029 MBB = StartMBB; 2030 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2031 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2032 MBB->addSuccessor(JoinMBB); 2033 MBB->addSuccessor(FalseMBB); 2034 2035 // FalseMBB: 2036 // store %SrcReg, %Disp(%Index,%Base) 2037 // # fallthrough to JoinMBB 2038 MBB = FalseMBB; 2039 BuildMI(MBB, DL, TII->get(StoreOpcode)) 2040 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 2041 MBB->addSuccessor(JoinMBB); 2042 2043 MI->eraseFromParent(); 2044 return JoinMBB; 2045} 2046 2047// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 2048// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 2049// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 2050// BitSize is the width of the field in bits, or 0 if this is a partword 2051// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 2052// is one of the operands. Invert says whether the field should be 2053// inverted after performing BinOpcode (e.g. for NAND). 2054MachineBasicBlock * 2055SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 2056 MachineBasicBlock *MBB, 2057 unsigned BinOpcode, 2058 unsigned BitSize, 2059 bool Invert) const { 2060 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2061 MachineFunction &MF = *MBB->getParent(); 2062 MachineRegisterInfo &MRI = MF.getRegInfo(); 2063 bool IsSubWord = (BitSize < 32); 2064 2065 // Extract the operands. Base can be a register or a frame index. 2066 // Src2 can be a register or immediate. 2067 unsigned Dest = MI->getOperand(0).getReg(); 2068 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2069 int64_t Disp = MI->getOperand(2).getImm(); 2070 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2071 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2072 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2073 DebugLoc DL = MI->getDebugLoc(); 2074 if (IsSubWord) 2075 BitSize = MI->getOperand(6).getImm(); 2076 2077 // Subword operations use 32-bit registers. 2078 const TargetRegisterClass *RC = (BitSize <= 32 ? 2079 &SystemZ::GR32BitRegClass : 2080 &SystemZ::GR64BitRegClass); 2081 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2082 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2083 2084 // Get the right opcodes for the displacement. 2085 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2086 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2087 assert(LOpcode && CSOpcode && "Displacement out of range"); 2088 2089 // Create virtual registers for temporary results. 2090 unsigned OrigVal = MRI.createVirtualRegister(RC); 2091 unsigned OldVal = MRI.createVirtualRegister(RC); 2092 unsigned NewVal = (BinOpcode || IsSubWord ? 2093 MRI.createVirtualRegister(RC) : Src2.getReg()); 2094 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2095 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2096 2097 // Insert a basic block for the main loop. 2098 MachineBasicBlock *StartMBB = MBB; 2099 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2100 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2101 2102 // StartMBB: 2103 // ... 2104 // %OrigVal = L Disp(%Base) 2105 // # fall through to LoopMMB 2106 MBB = StartMBB; 2107 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2108 .addOperand(Base).addImm(Disp).addReg(0); 2109 MBB->addSuccessor(LoopMBB); 2110 2111 // LoopMBB: 2112 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2113 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2114 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2115 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2116 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2117 // JNE LoopMBB 2118 // # fall through to DoneMMB 2119 MBB = LoopMBB; 2120 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2121 .addReg(OrigVal).addMBB(StartMBB) 2122 .addReg(Dest).addMBB(LoopMBB); 2123 if (IsSubWord) 2124 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2125 .addReg(OldVal).addReg(BitShift).addImm(0); 2126 if (Invert) { 2127 // Perform the operation normally and then invert every bit of the field. 2128 unsigned Tmp = MRI.createVirtualRegister(RC); 2129 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2130 .addReg(RotatedOldVal).addOperand(Src2); 2131 if (BitSize < 32) 2132 // XILF with the upper BitSize bits set. 2133 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 2134 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 2135 else if (BitSize == 32) 2136 // XILF with every bit set. 2137 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 2138 .addReg(Tmp).addImm(~uint32_t(0)); 2139 else { 2140 // Use LCGR and add -1 to the result, which is more compact than 2141 // an XILF, XILH pair. 2142 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2143 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2144 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2145 .addReg(Tmp2).addImm(-1); 2146 } 2147 } else if (BinOpcode) 2148 // A simply binary operation. 2149 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2150 .addReg(RotatedOldVal).addOperand(Src2); 2151 else if (IsSubWord) 2152 // Use RISBG to rotate Src2 into position and use it to replace the 2153 // field in RotatedOldVal. 2154 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2155 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2156 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2157 if (IsSubWord) 2158 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2159 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2160 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2161 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2162 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2163 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2164 MBB->addSuccessor(LoopMBB); 2165 MBB->addSuccessor(DoneMBB); 2166 2167 MI->eraseFromParent(); 2168 return DoneMBB; 2169} 2170 2171// Implement EmitInstrWithCustomInserter for pseudo 2172// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2173// instruction that should be used to compare the current field with the 2174// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2175// for when the current field should be kept. BitSize is the width of 2176// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2177MachineBasicBlock * 2178SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2179 MachineBasicBlock *MBB, 2180 unsigned CompareOpcode, 2181 unsigned KeepOldMask, 2182 unsigned BitSize) const { 2183 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2184 MachineFunction &MF = *MBB->getParent(); 2185 MachineRegisterInfo &MRI = MF.getRegInfo(); 2186 bool IsSubWord = (BitSize < 32); 2187 2188 // Extract the operands. Base can be a register or a frame index. 2189 unsigned Dest = MI->getOperand(0).getReg(); 2190 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2191 int64_t Disp = MI->getOperand(2).getImm(); 2192 unsigned Src2 = MI->getOperand(3).getReg(); 2193 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2194 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2195 DebugLoc DL = MI->getDebugLoc(); 2196 if (IsSubWord) 2197 BitSize = MI->getOperand(6).getImm(); 2198 2199 // Subword operations use 32-bit registers. 2200 const TargetRegisterClass *RC = (BitSize <= 32 ? 2201 &SystemZ::GR32BitRegClass : 2202 &SystemZ::GR64BitRegClass); 2203 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2204 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2205 2206 // Get the right opcodes for the displacement. 2207 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2208 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2209 assert(LOpcode && CSOpcode && "Displacement out of range"); 2210 2211 // Create virtual registers for temporary results. 2212 unsigned OrigVal = MRI.createVirtualRegister(RC); 2213 unsigned OldVal = MRI.createVirtualRegister(RC); 2214 unsigned NewVal = MRI.createVirtualRegister(RC); 2215 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2216 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2217 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2218 2219 // Insert 3 basic blocks for the loop. 2220 MachineBasicBlock *StartMBB = MBB; 2221 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2222 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2223 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2224 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2225 2226 // StartMBB: 2227 // ... 2228 // %OrigVal = L Disp(%Base) 2229 // # fall through to LoopMMB 2230 MBB = StartMBB; 2231 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2232 .addOperand(Base).addImm(Disp).addReg(0); 2233 MBB->addSuccessor(LoopMBB); 2234 2235 // LoopMBB: 2236 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2237 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2238 // CompareOpcode %RotatedOldVal, %Src2 2239 // BRC KeepOldMask, UpdateMBB 2240 MBB = LoopMBB; 2241 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2242 .addReg(OrigVal).addMBB(StartMBB) 2243 .addReg(Dest).addMBB(UpdateMBB); 2244 if (IsSubWord) 2245 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2246 .addReg(OldVal).addReg(BitShift).addImm(0); 2247 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2248 .addReg(RotatedOldVal).addReg(Src2); 2249 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2250 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2251 MBB->addSuccessor(UpdateMBB); 2252 MBB->addSuccessor(UseAltMBB); 2253 2254 // UseAltMBB: 2255 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2256 // # fall through to UpdateMMB 2257 MBB = UseAltMBB; 2258 if (IsSubWord) 2259 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2260 .addReg(RotatedOldVal).addReg(Src2) 2261 .addImm(32).addImm(31 + BitSize).addImm(0); 2262 MBB->addSuccessor(UpdateMBB); 2263 2264 // UpdateMBB: 2265 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2266 // [ %RotatedAltVal, UseAltMBB ] 2267 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2268 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2269 // JNE LoopMBB 2270 // # fall through to DoneMMB 2271 MBB = UpdateMBB; 2272 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2273 .addReg(RotatedOldVal).addMBB(LoopMBB) 2274 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2275 if (IsSubWord) 2276 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2277 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2278 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2279 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2280 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2281 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2282 MBB->addSuccessor(LoopMBB); 2283 MBB->addSuccessor(DoneMBB); 2284 2285 MI->eraseFromParent(); 2286 return DoneMBB; 2287} 2288 2289// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2290// instruction MI. 2291MachineBasicBlock * 2292SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2293 MachineBasicBlock *MBB) const { 2294 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2295 MachineFunction &MF = *MBB->getParent(); 2296 MachineRegisterInfo &MRI = MF.getRegInfo(); 2297 2298 // Extract the operands. Base can be a register or a frame index. 2299 unsigned Dest = MI->getOperand(0).getReg(); 2300 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2301 int64_t Disp = MI->getOperand(2).getImm(); 2302 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2303 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2304 unsigned BitShift = MI->getOperand(5).getReg(); 2305 unsigned NegBitShift = MI->getOperand(6).getReg(); 2306 int64_t BitSize = MI->getOperand(7).getImm(); 2307 DebugLoc DL = MI->getDebugLoc(); 2308 2309 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2310 2311 // Get the right opcodes for the displacement. 2312 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2313 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2314 assert(LOpcode && CSOpcode && "Displacement out of range"); 2315 2316 // Create virtual registers for temporary results. 2317 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2318 unsigned OldVal = MRI.createVirtualRegister(RC); 2319 unsigned CmpVal = MRI.createVirtualRegister(RC); 2320 unsigned SwapVal = MRI.createVirtualRegister(RC); 2321 unsigned StoreVal = MRI.createVirtualRegister(RC); 2322 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2323 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2324 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2325 2326 // Insert 2 basic blocks for the loop. 2327 MachineBasicBlock *StartMBB = MBB; 2328 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2329 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2330 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 2331 2332 // StartMBB: 2333 // ... 2334 // %OrigOldVal = L Disp(%Base) 2335 // # fall through to LoopMMB 2336 MBB = StartMBB; 2337 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 2338 .addOperand(Base).addImm(Disp).addReg(0); 2339 MBB->addSuccessor(LoopMBB); 2340 2341 // LoopMBB: 2342 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 2343 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 2344 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 2345 // %Dest = RLL %OldVal, BitSize(%BitShift) 2346 // ^^ The low BitSize bits contain the field 2347 // of interest. 2348 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 2349 // ^^ Replace the upper 32-BitSize bits of the 2350 // comparison value with those that we loaded, 2351 // so that we can use a full word comparison. 2352 // CR %Dest, %RetryCmpVal 2353 // JNE DoneMBB 2354 // # Fall through to SetMBB 2355 MBB = LoopMBB; 2356 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2357 .addReg(OrigOldVal).addMBB(StartMBB) 2358 .addReg(RetryOldVal).addMBB(SetMBB); 2359 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 2360 .addReg(OrigCmpVal).addMBB(StartMBB) 2361 .addReg(RetryCmpVal).addMBB(SetMBB); 2362 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 2363 .addReg(OrigSwapVal).addMBB(StartMBB) 2364 .addReg(RetrySwapVal).addMBB(SetMBB); 2365 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 2366 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 2367 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 2368 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2369 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 2370 .addReg(Dest).addReg(RetryCmpVal); 2371 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2372 .addImm(SystemZ::CCMASK_ICMP) 2373 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 2374 MBB->addSuccessor(DoneMBB); 2375 MBB->addSuccessor(SetMBB); 2376 2377 // SetMBB: 2378 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 2379 // ^^ Replace the upper 32-BitSize bits of the new 2380 // value with those that we loaded. 2381 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 2382 // ^^ Rotate the new field to its proper position. 2383 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 2384 // JNE LoopMBB 2385 // # fall through to ExitMMB 2386 MBB = SetMBB; 2387 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 2388 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2389 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 2390 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 2391 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2392 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2393 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2394 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2395 MBB->addSuccessor(LoopMBB); 2396 MBB->addSuccessor(DoneMBB); 2397 2398 MI->eraseFromParent(); 2399 return DoneMBB; 2400} 2401 2402// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2403// if the high register of the GR128 value must be cleared or false if 2404// it's "don't care". SubReg is subreg_odd32 when extending a GR32 2405// and subreg_odd when extending a GR64. 2406MachineBasicBlock * 2407SystemZTargetLowering::emitExt128(MachineInstr *MI, 2408 MachineBasicBlock *MBB, 2409 bool ClearEven, unsigned SubReg) const { 2410 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2411 MachineFunction &MF = *MBB->getParent(); 2412 MachineRegisterInfo &MRI = MF.getRegInfo(); 2413 DebugLoc DL = MI->getDebugLoc(); 2414 2415 unsigned Dest = MI->getOperand(0).getReg(); 2416 unsigned Src = MI->getOperand(1).getReg(); 2417 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2418 2419 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2420 if (ClearEven) { 2421 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2422 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2423 2424 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2425 .addImm(0); 2426 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2427 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_high); 2428 In128 = NewIn128; 2429 } 2430 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2431 .addReg(In128).addReg(Src).addImm(SubReg); 2432 2433 MI->eraseFromParent(); 2434 return MBB; 2435} 2436 2437MachineBasicBlock * 2438SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 2439 MachineBasicBlock *MBB, 2440 unsigned Opcode) const { 2441 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2442 DebugLoc DL = MI->getDebugLoc(); 2443 2444 MachineOperand DestBase = MI->getOperand(0); 2445 uint64_t DestDisp = MI->getOperand(1).getImm(); 2446 MachineOperand SrcBase = MI->getOperand(2); 2447 uint64_t SrcDisp = MI->getOperand(3).getImm(); 2448 uint64_t Length = MI->getOperand(4).getImm(); 2449 2450 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 2451 .addOperand(DestBase).addImm(DestDisp).addImm(Length) 2452 .addOperand(SrcBase).addImm(SrcDisp); 2453 2454 MI->eraseFromParent(); 2455 return MBB; 2456} 2457 2458// Decompose string pseudo-instruction MI into a loop that continually performs 2459// Opcode until CC != 3. 2460MachineBasicBlock * 2461SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 2462 MachineBasicBlock *MBB, 2463 unsigned Opcode) const { 2464 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2465 MachineFunction &MF = *MBB->getParent(); 2466 MachineRegisterInfo &MRI = MF.getRegInfo(); 2467 DebugLoc DL = MI->getDebugLoc(); 2468 2469 uint64_t End1Reg = MI->getOperand(0).getReg(); 2470 uint64_t Start1Reg = MI->getOperand(1).getReg(); 2471 uint64_t Start2Reg = MI->getOperand(2).getReg(); 2472 uint64_t CharReg = MI->getOperand(3).getReg(); 2473 2474 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 2475 uint64_t This1Reg = MRI.createVirtualRegister(RC); 2476 uint64_t This2Reg = MRI.createVirtualRegister(RC); 2477 uint64_t End2Reg = MRI.createVirtualRegister(RC); 2478 2479 MachineBasicBlock *StartMBB = MBB; 2480 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2481 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2482 2483 // StartMBB: 2484 // # fall through to LoopMMB 2485 MBB->addSuccessor(LoopMBB); 2486 2487 // LoopMBB: 2488 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 2489 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 2490 // R0W = %CharReg 2491 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0W 2492 // JO LoopMBB 2493 // # fall through to DoneMMB 2494 // 2495 // The load of R0W can be hoisted by post-RA LICM. 2496 MBB = LoopMBB; 2497 2498 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 2499 .addReg(Start1Reg).addMBB(StartMBB) 2500 .addReg(End1Reg).addMBB(LoopMBB); 2501 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 2502 .addReg(Start2Reg).addMBB(StartMBB) 2503 .addReg(End2Reg).addMBB(LoopMBB); 2504 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0W).addReg(CharReg); 2505 BuildMI(MBB, DL, TII->get(Opcode)) 2506 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 2507 .addReg(This1Reg).addReg(This2Reg); 2508 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2509 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 2510 MBB->addSuccessor(LoopMBB); 2511 MBB->addSuccessor(DoneMBB); 2512 2513 DoneMBB->addLiveIn(SystemZ::CC); 2514 2515 MI->eraseFromParent(); 2516 return DoneMBB; 2517} 2518 2519MachineBasicBlock *SystemZTargetLowering:: 2520EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 2521 switch (MI->getOpcode()) { 2522 case SystemZ::Select32: 2523 case SystemZ::SelectF32: 2524 case SystemZ::Select64: 2525 case SystemZ::SelectF64: 2526 case SystemZ::SelectF128: 2527 return emitSelect(MI, MBB); 2528 2529 case SystemZ::CondStore8_32: 2530 return emitCondStore(MI, MBB, SystemZ::STC32, 0, false); 2531 case SystemZ::CondStore8_32Inv: 2532 return emitCondStore(MI, MBB, SystemZ::STC32, 0, true); 2533 case SystemZ::CondStore16_32: 2534 return emitCondStore(MI, MBB, SystemZ::STH32, 0, false); 2535 case SystemZ::CondStore16_32Inv: 2536 return emitCondStore(MI, MBB, SystemZ::STH32, 0, true); 2537 case SystemZ::CondStore32_32: 2538 return emitCondStore(MI, MBB, SystemZ::ST32, SystemZ::STOC32, false); 2539 case SystemZ::CondStore32_32Inv: 2540 return emitCondStore(MI, MBB, SystemZ::ST32, SystemZ::STOC32, true); 2541 case SystemZ::CondStore8: 2542 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 2543 case SystemZ::CondStore8Inv: 2544 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 2545 case SystemZ::CondStore16: 2546 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 2547 case SystemZ::CondStore16Inv: 2548 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 2549 case SystemZ::CondStore32: 2550 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 2551 case SystemZ::CondStore32Inv: 2552 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 2553 case SystemZ::CondStore64: 2554 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 2555 case SystemZ::CondStore64Inv: 2556 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 2557 case SystemZ::CondStoreF32: 2558 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 2559 case SystemZ::CondStoreF32Inv: 2560 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 2561 case SystemZ::CondStoreF64: 2562 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 2563 case SystemZ::CondStoreF64Inv: 2564 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 2565 2566 case SystemZ::AEXT128_64: 2567 return emitExt128(MI, MBB, false, SystemZ::subreg_low); 2568 case SystemZ::ZEXT128_32: 2569 return emitExt128(MI, MBB, true, SystemZ::subreg_low32); 2570 case SystemZ::ZEXT128_64: 2571 return emitExt128(MI, MBB, true, SystemZ::subreg_low); 2572 2573 case SystemZ::ATOMIC_SWAPW: 2574 return emitAtomicLoadBinary(MI, MBB, 0, 0); 2575 case SystemZ::ATOMIC_SWAP_32: 2576 return emitAtomicLoadBinary(MI, MBB, 0, 32); 2577 case SystemZ::ATOMIC_SWAP_64: 2578 return emitAtomicLoadBinary(MI, MBB, 0, 64); 2579 2580 case SystemZ::ATOMIC_LOADW_AR: 2581 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 2582 case SystemZ::ATOMIC_LOADW_AFI: 2583 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 2584 case SystemZ::ATOMIC_LOAD_AR: 2585 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 2586 case SystemZ::ATOMIC_LOAD_AHI: 2587 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 2588 case SystemZ::ATOMIC_LOAD_AFI: 2589 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 2590 case SystemZ::ATOMIC_LOAD_AGR: 2591 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 2592 case SystemZ::ATOMIC_LOAD_AGHI: 2593 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 2594 case SystemZ::ATOMIC_LOAD_AGFI: 2595 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 2596 2597 case SystemZ::ATOMIC_LOADW_SR: 2598 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 2599 case SystemZ::ATOMIC_LOAD_SR: 2600 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 2601 case SystemZ::ATOMIC_LOAD_SGR: 2602 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 2603 2604 case SystemZ::ATOMIC_LOADW_NR: 2605 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 2606 case SystemZ::ATOMIC_LOADW_NILH: 2607 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0); 2608 case SystemZ::ATOMIC_LOAD_NR: 2609 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 2610 case SystemZ::ATOMIC_LOAD_NILL32: 2611 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32); 2612 case SystemZ::ATOMIC_LOAD_NILH32: 2613 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32); 2614 case SystemZ::ATOMIC_LOAD_NILF32: 2615 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32); 2616 case SystemZ::ATOMIC_LOAD_NGR: 2617 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 2618 case SystemZ::ATOMIC_LOAD_NILL: 2619 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64); 2620 case SystemZ::ATOMIC_LOAD_NILH: 2621 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64); 2622 case SystemZ::ATOMIC_LOAD_NIHL: 2623 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); 2624 case SystemZ::ATOMIC_LOAD_NIHH: 2625 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); 2626 case SystemZ::ATOMIC_LOAD_NILF: 2627 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64); 2628 case SystemZ::ATOMIC_LOAD_NIHF: 2629 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); 2630 2631 case SystemZ::ATOMIC_LOADW_OR: 2632 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 2633 case SystemZ::ATOMIC_LOADW_OILH: 2634 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 0); 2635 case SystemZ::ATOMIC_LOAD_OR: 2636 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 2637 case SystemZ::ATOMIC_LOAD_OILL32: 2638 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL32, 32); 2639 case SystemZ::ATOMIC_LOAD_OILH32: 2640 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 32); 2641 case SystemZ::ATOMIC_LOAD_OILF32: 2642 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF32, 32); 2643 case SystemZ::ATOMIC_LOAD_OGR: 2644 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 2645 case SystemZ::ATOMIC_LOAD_OILL: 2646 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 64); 2647 case SystemZ::ATOMIC_LOAD_OILH: 2648 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 64); 2649 case SystemZ::ATOMIC_LOAD_OIHL: 2650 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL, 64); 2651 case SystemZ::ATOMIC_LOAD_OIHH: 2652 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH, 64); 2653 case SystemZ::ATOMIC_LOAD_OILF: 2654 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 64); 2655 case SystemZ::ATOMIC_LOAD_OIHF: 2656 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF, 64); 2657 2658 case SystemZ::ATOMIC_LOADW_XR: 2659 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 2660 case SystemZ::ATOMIC_LOADW_XILF: 2661 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 0); 2662 case SystemZ::ATOMIC_LOAD_XR: 2663 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 2664 case SystemZ::ATOMIC_LOAD_XILF32: 2665 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 32); 2666 case SystemZ::ATOMIC_LOAD_XGR: 2667 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 2668 case SystemZ::ATOMIC_LOAD_XILF: 2669 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 64); 2670 case SystemZ::ATOMIC_LOAD_XIHF: 2671 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); 2672 2673 case SystemZ::ATOMIC_LOADW_NRi: 2674 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 2675 case SystemZ::ATOMIC_LOADW_NILHi: 2676 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0, true); 2677 case SystemZ::ATOMIC_LOAD_NRi: 2678 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 2679 case SystemZ::ATOMIC_LOAD_NILL32i: 2680 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32, true); 2681 case SystemZ::ATOMIC_LOAD_NILH32i: 2682 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32, true); 2683 case SystemZ::ATOMIC_LOAD_NILF32i: 2684 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32, true); 2685 case SystemZ::ATOMIC_LOAD_NGRi: 2686 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 2687 case SystemZ::ATOMIC_LOAD_NILLi: 2688 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64, true); 2689 case SystemZ::ATOMIC_LOAD_NILHi: 2690 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64, true); 2691 case SystemZ::ATOMIC_LOAD_NIHLi: 2692 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); 2693 case SystemZ::ATOMIC_LOAD_NIHHi: 2694 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); 2695 case SystemZ::ATOMIC_LOAD_NILFi: 2696 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64, true); 2697 case SystemZ::ATOMIC_LOAD_NIHFi: 2698 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); 2699 2700 case SystemZ::ATOMIC_LOADW_MIN: 2701 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2702 SystemZ::CCMASK_CMP_LE, 0); 2703 case SystemZ::ATOMIC_LOAD_MIN_32: 2704 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2705 SystemZ::CCMASK_CMP_LE, 32); 2706 case SystemZ::ATOMIC_LOAD_MIN_64: 2707 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2708 SystemZ::CCMASK_CMP_LE, 64); 2709 2710 case SystemZ::ATOMIC_LOADW_MAX: 2711 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2712 SystemZ::CCMASK_CMP_GE, 0); 2713 case SystemZ::ATOMIC_LOAD_MAX_32: 2714 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2715 SystemZ::CCMASK_CMP_GE, 32); 2716 case SystemZ::ATOMIC_LOAD_MAX_64: 2717 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2718 SystemZ::CCMASK_CMP_GE, 64); 2719 2720 case SystemZ::ATOMIC_LOADW_UMIN: 2721 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2722 SystemZ::CCMASK_CMP_LE, 0); 2723 case SystemZ::ATOMIC_LOAD_UMIN_32: 2724 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2725 SystemZ::CCMASK_CMP_LE, 32); 2726 case SystemZ::ATOMIC_LOAD_UMIN_64: 2727 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2728 SystemZ::CCMASK_CMP_LE, 64); 2729 2730 case SystemZ::ATOMIC_LOADW_UMAX: 2731 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2732 SystemZ::CCMASK_CMP_GE, 0); 2733 case SystemZ::ATOMIC_LOAD_UMAX_32: 2734 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2735 SystemZ::CCMASK_CMP_GE, 32); 2736 case SystemZ::ATOMIC_LOAD_UMAX_64: 2737 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2738 SystemZ::CCMASK_CMP_GE, 64); 2739 2740 case SystemZ::ATOMIC_CMP_SWAPW: 2741 return emitAtomicCmpSwapW(MI, MBB); 2742 case SystemZ::MVCWrapper: 2743 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 2744 case SystemZ::CLCWrapper: 2745 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 2746 case SystemZ::CLSTLoop: 2747 return emitStringWrapper(MI, MBB, SystemZ::CLST); 2748 case SystemZ::MVSTLoop: 2749 return emitStringWrapper(MI, MBB, SystemZ::MVST); 2750 case SystemZ::SRSTLoop: 2751 return emitStringWrapper(MI, MBB, SystemZ::SRST); 2752 default: 2753 llvm_unreachable("Unexpected instr type to insert"); 2754 } 2755} 2756