SystemZISelLowering.cpp revision df40f8e8ad0aa93defa44b8a136e8d871cfd44ea
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "systemz-lower" 15 16#include "SystemZISelLowering.h" 17#include "SystemZCallingConv.h" 18#include "SystemZConstantPoolValue.h" 19#include "SystemZMachineFunctionInfo.h" 20#include "SystemZTargetMachine.h" 21#include "llvm/CodeGen/CallingConvLower.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25 26using namespace llvm; 27 28// Classify VT as either 32 or 64 bit. 29static bool is32Bit(EVT VT) { 30 switch (VT.getSimpleVT().SimpleTy) { 31 case MVT::i32: 32 return true; 33 case MVT::i64: 34 return false; 35 default: 36 llvm_unreachable("Unsupported type"); 37 } 38} 39 40// Return a version of MachineOperand that can be safely used before the 41// final use. 42static MachineOperand earlyUseOperand(MachineOperand Op) { 43 if (Op.isReg()) 44 Op.setIsKill(false); 45 return Op; 46} 47 48SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 49 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 50 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 51 MVT PtrVT = getPointerTy(); 52 53 // Set up the register classes. 54 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 55 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 56 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 57 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 58 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 59 60 // Compute derived properties from the register classes 61 computeRegisterProperties(); 62 63 // Set up special registers. 64 setExceptionPointerRegister(SystemZ::R6D); 65 setExceptionSelectorRegister(SystemZ::R7D); 66 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 67 68 // TODO: It may be better to default to latency-oriented scheduling, however 69 // LLVM's current latency-oriented scheduler can't handle physreg definitions 70 // such as SystemZ has with CC, so set this to the register-pressure 71 // scheduler, because it can. 72 setSchedulingPreference(Sched::RegPressure); 73 74 setBooleanContents(ZeroOrOneBooleanContent); 75 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 76 77 // Instructions are strings of 2-byte aligned 2-byte values. 78 setMinFunctionAlignment(2); 79 80 // Handle operations that are handled in a similar way for all types. 81 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 82 I <= MVT::LAST_FP_VALUETYPE; 83 ++I) { 84 MVT VT = MVT::SimpleValueType(I); 85 if (isTypeLegal(VT)) { 86 // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). 87 setOperationAction(ISD::SETCC, VT, Expand); 88 89 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 90 setOperationAction(ISD::SELECT, VT, Expand); 91 92 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 93 setOperationAction(ISD::SELECT_CC, VT, Custom); 94 setOperationAction(ISD::BR_CC, VT, Custom); 95 } 96 } 97 98 // Expand jump table branches as address arithmetic followed by an 99 // indirect jump. 100 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 101 102 // Expand BRCOND into a BR_CC (see above). 103 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 104 105 // Handle integer types. 106 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 107 I <= MVT::LAST_INTEGER_VALUETYPE; 108 ++I) { 109 MVT VT = MVT::SimpleValueType(I); 110 if (isTypeLegal(VT)) { 111 // Expand individual DIV and REMs into DIVREMs. 112 setOperationAction(ISD::SDIV, VT, Expand); 113 setOperationAction(ISD::UDIV, VT, Expand); 114 setOperationAction(ISD::SREM, VT, Expand); 115 setOperationAction(ISD::UREM, VT, Expand); 116 setOperationAction(ISD::SDIVREM, VT, Custom); 117 setOperationAction(ISD::UDIVREM, VT, Custom); 118 119 // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 120 // FIXME: probably much too conservative. 121 setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 122 setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 123 124 // No special instructions for these. 125 setOperationAction(ISD::CTPOP, VT, Expand); 126 setOperationAction(ISD::CTTZ, VT, Expand); 127 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 128 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 129 setOperationAction(ISD::ROTR, VT, Expand); 130 131 // Use *MUL_LOHI where possible instead of MULH*. 132 setOperationAction(ISD::MULHS, VT, Expand); 133 setOperationAction(ISD::MULHU, VT, Expand); 134 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 135 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 136 137 // We have instructions for signed but not unsigned FP conversion. 138 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 139 } 140 } 141 142 // Type legalization will convert 8- and 16-bit atomic operations into 143 // forms that operate on i32s (but still keeping the original memory VT). 144 // Lower them into full i32 operations. 145 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 146 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 147 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 148 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 149 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 150 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 151 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 152 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 153 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 154 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 155 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 156 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 157 158 // We have instructions for signed but not unsigned FP conversion. 159 // Handle unsigned 32-bit types as signed 64-bit types. 160 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 161 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 162 163 // We have native support for a 64-bit CTLZ, via FLOGR. 164 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 165 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 166 167 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 168 setOperationAction(ISD::OR, MVT::i64, Custom); 169 170 // FIXME: Can we support these natively? 171 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 172 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 173 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 174 175 // We have native instructions for i8, i16 and i32 extensions, but not i1. 176 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 177 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 178 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 179 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 180 181 // Handle the various types of symbolic address. 182 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 183 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 184 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 185 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 186 setOperationAction(ISD::JumpTable, PtrVT, Custom); 187 188 // We need to handle dynamic allocations specially because of the 189 // 160-byte area at the bottom of the stack. 190 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 191 192 // Use custom expanders so that we can force the function to use 193 // a frame pointer. 194 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 195 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 196 197 // Handle floating-point types. 198 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 199 I <= MVT::LAST_FP_VALUETYPE; 200 ++I) { 201 MVT VT = MVT::SimpleValueType(I); 202 if (isTypeLegal(VT)) { 203 // We can use FI for FRINT. 204 setOperationAction(ISD::FRINT, VT, Legal); 205 206 // We can use the extended form of FI for other rounding operations. 207 if (Subtarget.hasFPExtension()) { 208 setOperationAction(ISD::FNEARBYINT, VT, Legal); 209 setOperationAction(ISD::FFLOOR, VT, Legal); 210 setOperationAction(ISD::FCEIL, VT, Legal); 211 setOperationAction(ISD::FTRUNC, VT, Legal); 212 setOperationAction(ISD::FROUND, VT, Legal); 213 } 214 215 // No special instructions for these. 216 setOperationAction(ISD::FSIN, VT, Expand); 217 setOperationAction(ISD::FCOS, VT, Expand); 218 setOperationAction(ISD::FREM, VT, Expand); 219 } 220 } 221 222 // We have fused multiply-addition for f32 and f64 but not f128. 223 setOperationAction(ISD::FMA, MVT::f32, Legal); 224 setOperationAction(ISD::FMA, MVT::f64, Legal); 225 setOperationAction(ISD::FMA, MVT::f128, Expand); 226 227 // Needed so that we don't try to implement f128 constant loads using 228 // a load-and-extend of a f80 constant (in cases where the constant 229 // would fit in an f80). 230 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 231 232 // Floating-point truncation and stores need to be done separately. 233 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 234 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 235 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 236 237 // We have 64-bit FPR<->GPR moves, but need special handling for 238 // 32-bit forms. 239 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 240 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 241 242 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 243 // structure, but VAEND is a no-op. 244 setOperationAction(ISD::VASTART, MVT::Other, Custom); 245 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 246 setOperationAction(ISD::VAEND, MVT::Other, Expand); 247 248 // We want to use MVC in preference to even a single load/store pair. 249 MaxStoresPerMemcpy = 0; 250 MaxStoresPerMemcpyOptSize = 0; 251 252 // The main memset sequence is a byte store followed by an MVC. 253 // Two STC or MV..I stores win over that, but the kind of fused stores 254 // generated by target-independent code don't when the byte value is 255 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 256 // than "STC;MVC". Handle the choice in target-specific code instead. 257 MaxStoresPerMemset = 0; 258 MaxStoresPerMemsetOptSize = 0; 259} 260 261bool 262SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 263 VT = VT.getScalarType(); 264 265 if (!VT.isSimple()) 266 return false; 267 268 switch (VT.getSimpleVT().SimpleTy) { 269 case MVT::f32: 270 case MVT::f64: 271 return true; 272 case MVT::f128: 273 return false; 274 default: 275 break; 276 } 277 278 return false; 279} 280 281bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 282 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 283 return Imm.isZero() || Imm.isNegZero(); 284} 285 286bool SystemZTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 287 bool *Fast) const { 288 // Unaligned accesses should never be slower than the expanded version. 289 // We check specifically for aligned accesses in the few cases where 290 // they are required. 291 if (Fast) 292 *Fast = true; 293 return true; 294} 295 296bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 297 Type *Ty) const { 298 // Punt on globals for now, although they can be used in limited 299 // RELATIVE LONG cases. 300 if (AM.BaseGV) 301 return false; 302 303 // Require a 20-bit signed offset. 304 if (!isInt<20>(AM.BaseOffs)) 305 return false; 306 307 // Indexing is OK but no scale factor can be applied. 308 return AM.Scale == 0 || AM.Scale == 1; 309} 310 311bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 312 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 313 return false; 314 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 315 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 316 return FromBits > ToBits; 317} 318 319bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 320 if (!FromVT.isInteger() || !ToVT.isInteger()) 321 return false; 322 unsigned FromBits = FromVT.getSizeInBits(); 323 unsigned ToBits = ToVT.getSizeInBits(); 324 return FromBits > ToBits; 325} 326 327//===----------------------------------------------------------------------===// 328// Inline asm support 329//===----------------------------------------------------------------------===// 330 331TargetLowering::ConstraintType 332SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 333 if (Constraint.size() == 1) { 334 switch (Constraint[0]) { 335 case 'a': // Address register 336 case 'd': // Data register (equivalent to 'r') 337 case 'f': // Floating-point register 338 case 'r': // General-purpose register 339 return C_RegisterClass; 340 341 case 'Q': // Memory with base and unsigned 12-bit displacement 342 case 'R': // Likewise, plus an index 343 case 'S': // Memory with base and signed 20-bit displacement 344 case 'T': // Likewise, plus an index 345 case 'm': // Equivalent to 'T'. 346 return C_Memory; 347 348 case 'I': // Unsigned 8-bit constant 349 case 'J': // Unsigned 12-bit constant 350 case 'K': // Signed 16-bit constant 351 case 'L': // Signed 20-bit displacement (on all targets we support) 352 case 'M': // 0x7fffffff 353 return C_Other; 354 355 default: 356 break; 357 } 358 } 359 return TargetLowering::getConstraintType(Constraint); 360} 361 362TargetLowering::ConstraintWeight SystemZTargetLowering:: 363getSingleConstraintMatchWeight(AsmOperandInfo &info, 364 const char *constraint) const { 365 ConstraintWeight weight = CW_Invalid; 366 Value *CallOperandVal = info.CallOperandVal; 367 // If we don't have a value, we can't do a match, 368 // but allow it at the lowest weight. 369 if (CallOperandVal == NULL) 370 return CW_Default; 371 Type *type = CallOperandVal->getType(); 372 // Look at the constraint type. 373 switch (*constraint) { 374 default: 375 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 376 break; 377 378 case 'a': // Address register 379 case 'd': // Data register (equivalent to 'r') 380 case 'r': // General-purpose register 381 if (CallOperandVal->getType()->isIntegerTy()) 382 weight = CW_Register; 383 break; 384 385 case 'f': // Floating-point register 386 if (type->isFloatingPointTy()) 387 weight = CW_Register; 388 break; 389 390 case 'I': // Unsigned 8-bit constant 391 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 392 if (isUInt<8>(C->getZExtValue())) 393 weight = CW_Constant; 394 break; 395 396 case 'J': // Unsigned 12-bit constant 397 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 398 if (isUInt<12>(C->getZExtValue())) 399 weight = CW_Constant; 400 break; 401 402 case 'K': // Signed 16-bit constant 403 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 404 if (isInt<16>(C->getSExtValue())) 405 weight = CW_Constant; 406 break; 407 408 case 'L': // Signed 20-bit displacement (on all targets we support) 409 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 410 if (isInt<20>(C->getSExtValue())) 411 weight = CW_Constant; 412 break; 413 414 case 'M': // 0x7fffffff 415 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 416 if (C->getZExtValue() == 0x7fffffff) 417 weight = CW_Constant; 418 break; 419 } 420 return weight; 421} 422 423// Parse a "{tNNN}" register constraint for which the register type "t" 424// has already been verified. MC is the class associated with "t" and 425// Map maps 0-based register numbers to LLVM register numbers. 426static std::pair<unsigned, const TargetRegisterClass *> 427parseRegisterNumber(const std::string &Constraint, 428 const TargetRegisterClass *RC, const unsigned *Map) { 429 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 430 if (isdigit(Constraint[2])) { 431 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 432 unsigned Index = atoi(Suffix.c_str()); 433 if (Index < 16 && Map[Index]) 434 return std::make_pair(Map[Index], RC); 435 } 436 return std::make_pair(0u, static_cast<TargetRegisterClass*>(0)); 437} 438 439std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 440getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 441 if (Constraint.size() == 1) { 442 // GCC Constraint Letters 443 switch (Constraint[0]) { 444 default: break; 445 case 'd': // Data register (equivalent to 'r') 446 case 'r': // General-purpose register 447 if (VT == MVT::i64) 448 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 449 else if (VT == MVT::i128) 450 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 451 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 452 453 case 'a': // Address register 454 if (VT == MVT::i64) 455 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 456 else if (VT == MVT::i128) 457 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 458 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 459 460 case 'f': // Floating-point register 461 if (VT == MVT::f64) 462 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 463 else if (VT == MVT::f128) 464 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 465 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 466 } 467 } 468 if (Constraint[0] == '{') { 469 // We need to override the default register parsing for GPRs and FPRs 470 // because the interpretation depends on VT. The internal names of 471 // the registers are also different from the external names 472 // (F0D and F0S instead of F0, etc.). 473 if (Constraint[1] == 'r') { 474 if (VT == MVT::i32) 475 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 476 SystemZMC::GR32Regs); 477 if (VT == MVT::i128) 478 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 479 SystemZMC::GR128Regs); 480 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 481 SystemZMC::GR64Regs); 482 } 483 if (Constraint[1] == 'f') { 484 if (VT == MVT::f32) 485 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 486 SystemZMC::FP32Regs); 487 if (VT == MVT::f128) 488 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 489 SystemZMC::FP128Regs); 490 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 491 SystemZMC::FP64Regs); 492 } 493 } 494 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 495} 496 497void SystemZTargetLowering:: 498LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 499 std::vector<SDValue> &Ops, 500 SelectionDAG &DAG) const { 501 // Only support length 1 constraints for now. 502 if (Constraint.length() == 1) { 503 switch (Constraint[0]) { 504 case 'I': // Unsigned 8-bit constant 505 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 506 if (isUInt<8>(C->getZExtValue())) 507 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 508 Op.getValueType())); 509 return; 510 511 case 'J': // Unsigned 12-bit constant 512 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 513 if (isUInt<12>(C->getZExtValue())) 514 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 515 Op.getValueType())); 516 return; 517 518 case 'K': // Signed 16-bit constant 519 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 520 if (isInt<16>(C->getSExtValue())) 521 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 522 Op.getValueType())); 523 return; 524 525 case 'L': // Signed 20-bit displacement (on all targets we support) 526 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 527 if (isInt<20>(C->getSExtValue())) 528 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 529 Op.getValueType())); 530 return; 531 532 case 'M': // 0x7fffffff 533 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 534 if (C->getZExtValue() == 0x7fffffff) 535 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 536 Op.getValueType())); 537 return; 538 } 539 } 540 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 541} 542 543//===----------------------------------------------------------------------===// 544// Calling conventions 545//===----------------------------------------------------------------------===// 546 547#include "SystemZGenCallingConv.inc" 548 549bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 550 Type *ToType) const { 551 return isTruncateFree(FromType, ToType); 552} 553 554bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 555 if (!CI->isTailCall()) 556 return false; 557 return true; 558} 559 560// Value is a value that has been passed to us in the location described by VA 561// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 562// any loads onto Chain. 563static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 564 CCValAssign &VA, SDValue Chain, 565 SDValue Value) { 566 // If the argument has been promoted from a smaller type, insert an 567 // assertion to capture this. 568 if (VA.getLocInfo() == CCValAssign::SExt) 569 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 570 DAG.getValueType(VA.getValVT())); 571 else if (VA.getLocInfo() == CCValAssign::ZExt) 572 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 573 DAG.getValueType(VA.getValVT())); 574 575 if (VA.isExtInLoc()) 576 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 577 else if (VA.getLocInfo() == CCValAssign::Indirect) 578 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 579 MachinePointerInfo(), false, false, false, 0); 580 else 581 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 582 return Value; 583} 584 585// Value is a value of type VA.getValVT() that we need to copy into 586// the location described by VA. Return a copy of Value converted to 587// VA.getValVT(). The caller is responsible for handling indirect values. 588static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 589 CCValAssign &VA, SDValue Value) { 590 switch (VA.getLocInfo()) { 591 case CCValAssign::SExt: 592 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 593 case CCValAssign::ZExt: 594 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 595 case CCValAssign::AExt: 596 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 597 case CCValAssign::Full: 598 return Value; 599 default: 600 llvm_unreachable("Unhandled getLocInfo()"); 601 } 602} 603 604SDValue SystemZTargetLowering:: 605LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 606 const SmallVectorImpl<ISD::InputArg> &Ins, 607 SDLoc DL, SelectionDAG &DAG, 608 SmallVectorImpl<SDValue> &InVals) const { 609 MachineFunction &MF = DAG.getMachineFunction(); 610 MachineFrameInfo *MFI = MF.getFrameInfo(); 611 MachineRegisterInfo &MRI = MF.getRegInfo(); 612 SystemZMachineFunctionInfo *FuncInfo = 613 MF.getInfo<SystemZMachineFunctionInfo>(); 614 const SystemZFrameLowering *TFL = 615 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 616 617 // Assign locations to all of the incoming arguments. 618 SmallVector<CCValAssign, 16> ArgLocs; 619 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 620 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 621 622 unsigned NumFixedGPRs = 0; 623 unsigned NumFixedFPRs = 0; 624 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 625 SDValue ArgValue; 626 CCValAssign &VA = ArgLocs[I]; 627 EVT LocVT = VA.getLocVT(); 628 if (VA.isRegLoc()) { 629 // Arguments passed in registers 630 const TargetRegisterClass *RC; 631 switch (LocVT.getSimpleVT().SimpleTy) { 632 default: 633 // Integers smaller than i64 should be promoted to i64. 634 llvm_unreachable("Unexpected argument type"); 635 case MVT::i32: 636 NumFixedGPRs += 1; 637 RC = &SystemZ::GR32BitRegClass; 638 break; 639 case MVT::i64: 640 NumFixedGPRs += 1; 641 RC = &SystemZ::GR64BitRegClass; 642 break; 643 case MVT::f32: 644 NumFixedFPRs += 1; 645 RC = &SystemZ::FP32BitRegClass; 646 break; 647 case MVT::f64: 648 NumFixedFPRs += 1; 649 RC = &SystemZ::FP64BitRegClass; 650 break; 651 } 652 653 unsigned VReg = MRI.createVirtualRegister(RC); 654 MRI.addLiveIn(VA.getLocReg(), VReg); 655 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 656 } else { 657 assert(VA.isMemLoc() && "Argument not register or memory"); 658 659 // Create the frame index object for this incoming parameter. 660 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 661 VA.getLocMemOffset(), true); 662 663 // Create the SelectionDAG nodes corresponding to a load 664 // from this parameter. Unpromoted ints and floats are 665 // passed as right-justified 8-byte values. 666 EVT PtrVT = getPointerTy(); 667 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 668 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 669 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 670 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 671 MachinePointerInfo::getFixedStack(FI), 672 false, false, false, 0); 673 } 674 675 // Convert the value of the argument register into the value that's 676 // being passed. 677 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 678 } 679 680 if (IsVarArg) { 681 // Save the number of non-varargs registers for later use by va_start, etc. 682 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 683 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 684 685 // Likewise the address (in the form of a frame index) of where the 686 // first stack vararg would be. The 1-byte size here is arbitrary. 687 int64_t StackSize = CCInfo.getNextStackOffset(); 688 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 689 690 // ...and a similar frame index for the caller-allocated save area 691 // that will be used to store the incoming registers. 692 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 693 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 694 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 695 696 // Store the FPR varargs in the reserved frame slots. (We store the 697 // GPRs as part of the prologue.) 698 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 699 SDValue MemOps[SystemZ::NumArgFPRs]; 700 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 701 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 702 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 703 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 704 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 705 &SystemZ::FP64BitRegClass); 706 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 707 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 708 MachinePointerInfo::getFixedStack(FI), 709 false, false, 0); 710 711 } 712 // Join the stores, which are independent of one another. 713 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 714 &MemOps[NumFixedFPRs], 715 SystemZ::NumArgFPRs - NumFixedFPRs); 716 } 717 } 718 719 return Chain; 720} 721 722static bool canUseSiblingCall(CCState ArgCCInfo, 723 SmallVectorImpl<CCValAssign> &ArgLocs) { 724 // Punt if there are any indirect or stack arguments, or if the call 725 // needs the call-saved argument register R6. 726 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 727 CCValAssign &VA = ArgLocs[I]; 728 if (VA.getLocInfo() == CCValAssign::Indirect) 729 return false; 730 if (!VA.isRegLoc()) 731 return false; 732 unsigned Reg = VA.getLocReg(); 733 if (Reg == SystemZ::R6W || Reg == SystemZ::R6D) 734 return false; 735 } 736 return true; 737} 738 739SDValue 740SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 741 SmallVectorImpl<SDValue> &InVals) const { 742 SelectionDAG &DAG = CLI.DAG; 743 SDLoc &DL = CLI.DL; 744 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 745 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 746 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 747 SDValue Chain = CLI.Chain; 748 SDValue Callee = CLI.Callee; 749 bool &IsTailCall = CLI.IsTailCall; 750 CallingConv::ID CallConv = CLI.CallConv; 751 bool IsVarArg = CLI.IsVarArg; 752 MachineFunction &MF = DAG.getMachineFunction(); 753 EVT PtrVT = getPointerTy(); 754 755 // Analyze the operands of the call, assigning locations to each operand. 756 SmallVector<CCValAssign, 16> ArgLocs; 757 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 758 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 759 760 // We don't support GuaranteedTailCallOpt, only automatically-detected 761 // sibling calls. 762 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 763 IsTailCall = false; 764 765 // Get a count of how many bytes are to be pushed on the stack. 766 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 767 768 // Mark the start of the call. 769 if (!IsTailCall) 770 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 771 DL); 772 773 // Copy argument values to their designated locations. 774 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 775 SmallVector<SDValue, 8> MemOpChains; 776 SDValue StackPtr; 777 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 778 CCValAssign &VA = ArgLocs[I]; 779 SDValue ArgValue = OutVals[I]; 780 781 if (VA.getLocInfo() == CCValAssign::Indirect) { 782 // Store the argument in a stack slot and pass its address. 783 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 784 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 785 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 786 MachinePointerInfo::getFixedStack(FI), 787 false, false, 0)); 788 ArgValue = SpillSlot; 789 } else 790 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 791 792 if (VA.isRegLoc()) 793 // Queue up the argument copies and emit them at the end. 794 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 795 else { 796 assert(VA.isMemLoc() && "Argument not register or memory"); 797 798 // Work out the address of the stack slot. Unpromoted ints and 799 // floats are passed as right-justified 8-byte values. 800 if (!StackPtr.getNode()) 801 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 802 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 803 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 804 Offset += 4; 805 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 806 DAG.getIntPtrConstant(Offset)); 807 808 // Emit the store. 809 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 810 MachinePointerInfo(), 811 false, false, 0)); 812 } 813 } 814 815 // Join the stores, which are independent of one another. 816 if (!MemOpChains.empty()) 817 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 818 &MemOpChains[0], MemOpChains.size()); 819 820 // Accept direct calls by converting symbolic call addresses to the 821 // associated Target* opcodes. Force %r1 to be used for indirect 822 // tail calls. 823 SDValue Glue; 824 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 825 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 826 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 827 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 828 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 829 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 830 } else if (IsTailCall) { 831 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 832 Glue = Chain.getValue(1); 833 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 834 } 835 836 // Build a sequence of copy-to-reg nodes, chained and glued together. 837 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 838 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 839 RegsToPass[I].second, Glue); 840 Glue = Chain.getValue(1); 841 } 842 843 // The first call operand is the chain and the second is the target address. 844 SmallVector<SDValue, 8> Ops; 845 Ops.push_back(Chain); 846 Ops.push_back(Callee); 847 848 // Add argument registers to the end of the list so that they are 849 // known live into the call. 850 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 851 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 852 RegsToPass[I].second.getValueType())); 853 854 // Glue the call to the argument copies, if any. 855 if (Glue.getNode()) 856 Ops.push_back(Glue); 857 858 // Emit the call. 859 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 860 if (IsTailCall) 861 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, &Ops[0], Ops.size()); 862 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 863 Glue = Chain.getValue(1); 864 865 // Mark the end of the call, which is glued to the call itself. 866 Chain = DAG.getCALLSEQ_END(Chain, 867 DAG.getConstant(NumBytes, PtrVT, true), 868 DAG.getConstant(0, PtrVT, true), 869 Glue, DL); 870 Glue = Chain.getValue(1); 871 872 // Assign locations to each value returned by this call. 873 SmallVector<CCValAssign, 16> RetLocs; 874 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 875 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 876 877 // Copy all of the result registers out of their specified physreg. 878 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 879 CCValAssign &VA = RetLocs[I]; 880 881 // Copy the value out, gluing the copy to the end of the call sequence. 882 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 883 VA.getLocVT(), Glue); 884 Chain = RetValue.getValue(1); 885 Glue = RetValue.getValue(2); 886 887 // Convert the value of the return register into the value that's 888 // being returned. 889 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 890 } 891 892 return Chain; 893} 894 895SDValue 896SystemZTargetLowering::LowerReturn(SDValue Chain, 897 CallingConv::ID CallConv, bool IsVarArg, 898 const SmallVectorImpl<ISD::OutputArg> &Outs, 899 const SmallVectorImpl<SDValue> &OutVals, 900 SDLoc DL, SelectionDAG &DAG) const { 901 MachineFunction &MF = DAG.getMachineFunction(); 902 903 // Assign locations to each returned value. 904 SmallVector<CCValAssign, 16> RetLocs; 905 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 906 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 907 908 // Quick exit for void returns 909 if (RetLocs.empty()) 910 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 911 912 // Copy the result values into the output registers. 913 SDValue Glue; 914 SmallVector<SDValue, 4> RetOps; 915 RetOps.push_back(Chain); 916 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 917 CCValAssign &VA = RetLocs[I]; 918 SDValue RetValue = OutVals[I]; 919 920 // Make the return register live on exit. 921 assert(VA.isRegLoc() && "Can only return in registers!"); 922 923 // Promote the value as required. 924 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 925 926 // Chain and glue the copies together. 927 unsigned Reg = VA.getLocReg(); 928 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 929 Glue = Chain.getValue(1); 930 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 931 } 932 933 // Update chain and glue. 934 RetOps[0] = Chain; 935 if (Glue.getNode()) 936 RetOps.push_back(Glue); 937 938 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 939 RetOps.data(), RetOps.size()); 940} 941 942// CC is a comparison that will be implemented using an integer or 943// floating-point comparison. Return the condition code mask for 944// a branch on true. In the integer case, CCMASK_CMP_UO is set for 945// unsigned comparisons and clear for signed ones. In the floating-point 946// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 947static unsigned CCMaskForCondCode(ISD::CondCode CC) { 948#define CONV(X) \ 949 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 950 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 951 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 952 953 switch (CC) { 954 default: 955 llvm_unreachable("Invalid integer condition!"); 956 957 CONV(EQ); 958 CONV(NE); 959 CONV(GT); 960 CONV(GE); 961 CONV(LT); 962 CONV(LE); 963 964 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 965 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 966 } 967#undef CONV 968} 969 970// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 971// can be converted to a comparison against zero, adjust the operands 972// as necessary. 973static void adjustZeroCmp(SelectionDAG &DAG, bool &IsUnsigned, 974 SDValue &CmpOp0, SDValue &CmpOp1, 975 unsigned &CCMask) { 976 if (IsUnsigned) 977 return; 978 979 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(CmpOp1.getNode()); 980 if (!ConstOp1) 981 return; 982 983 int64_t Value = ConstOp1->getSExtValue(); 984 if ((Value == -1 && CCMask == SystemZ::CCMASK_CMP_GT) || 985 (Value == -1 && CCMask == SystemZ::CCMASK_CMP_LE) || 986 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_LT) || 987 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_GE)) { 988 CCMask ^= SystemZ::CCMASK_CMP_EQ; 989 CmpOp1 = DAG.getConstant(0, CmpOp1.getValueType()); 990 } 991} 992 993// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 994// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 995static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 996 SDValue &CmpOp0, SDValue &CmpOp1, 997 unsigned &CCMask) { 998 // For us to make any changes, it must a comparison between a single-use 999 // load and a constant. 1000 if (!CmpOp0.hasOneUse() || 1001 CmpOp0.getOpcode() != ISD::LOAD || 1002 CmpOp1.getOpcode() != ISD::Constant) 1003 return; 1004 1005 // We must have an 8- or 16-bit load. 1006 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 1007 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1008 if (NumBits != 8 && NumBits != 16) 1009 return; 1010 1011 // The load must be an extending one and the constant must be within the 1012 // range of the unextended value. 1013 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 1014 uint64_t Value = Constant->getZExtValue(); 1015 uint64_t Mask = (1 << NumBits) - 1; 1016 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1017 int64_t SignedValue = Constant->getSExtValue(); 1018 if (uint64_t(SignedValue) + (1ULL << (NumBits - 1)) > Mask) 1019 return; 1020 // Unsigned comparison between two sign-extended values is equivalent 1021 // to unsigned comparison between two zero-extended values. 1022 if (IsUnsigned) 1023 Value &= Mask; 1024 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 1025 CCMask == SystemZ::CCMASK_CMP_NE) 1026 // Any choice of IsUnsigned is OK for equality comparisons. 1027 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 1028 // but since we use CLHHSI for zero extensions, it seems better 1029 // to be consistent and do the same here. 1030 Value &= Mask, IsUnsigned = true; 1031 else if (NumBits == 8) { 1032 // Try to treat the comparison as unsigned, so that we can use CLI. 1033 // Adjust CCMask and Value as necessary. 1034 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 1035 // Test whether the high bit of the byte is set. 1036 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 1037 else if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_GE) 1038 // Test whether the high bit of the byte is clear. 1039 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 1040 else 1041 // No instruction exists for this combination. 1042 return; 1043 } 1044 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1045 if (Value > Mask) 1046 return; 1047 // Signed comparison between two zero-extended values is equivalent 1048 // to unsigned comparison. 1049 IsUnsigned = true; 1050 } else 1051 return; 1052 1053 // Make sure that the first operand is an i32 of the right extension type. 1054 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 1055 if (CmpOp0.getValueType() != MVT::i32 || 1056 Load->getExtensionType() != ExtType) 1057 CmpOp0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1058 Load->getChain(), Load->getBasePtr(), 1059 Load->getPointerInfo(), Load->getMemoryVT(), 1060 Load->isVolatile(), Load->isNonTemporal(), 1061 Load->getAlignment()); 1062 1063 // Make sure that the second operand is an i32 with the right value. 1064 if (CmpOp1.getValueType() != MVT::i32 || 1065 Value != Constant->getZExtValue()) 1066 CmpOp1 = DAG.getConstant(Value, MVT::i32); 1067} 1068 1069// Return true if a comparison described by CCMask, CmpOp0 and CmpOp1 1070// is an equality comparison that is better implemented using unsigned 1071// rather than signed comparison instructions. 1072static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0, 1073 SDValue CmpOp1, unsigned CCMask) { 1074 // The test must be for equality or inequality. 1075 if (CCMask != SystemZ::CCMASK_CMP_EQ && CCMask != SystemZ::CCMASK_CMP_NE) 1076 return false; 1077 1078 if (CmpOp1.getOpcode() == ISD::Constant) { 1079 uint64_t Value = cast<ConstantSDNode>(CmpOp1)->getSExtValue(); 1080 1081 // If we're comparing with memory, prefer unsigned comparisons for 1082 // values that are in the unsigned 16-bit range but not the signed 1083 // 16-bit range. We want to use CLFHSI and CLGHSI. 1084 if (CmpOp0.hasOneUse() && 1085 ISD::isNormalLoad(CmpOp0.getNode()) && 1086 (Value >= 32768 && Value < 65536)) 1087 return true; 1088 1089 // Use unsigned comparisons for values that are in the CLGFI range 1090 // but not in the CGFI range. 1091 if (CmpOp0.getValueType() == MVT::i64 && (Value >> 31) == 1) 1092 return true; 1093 1094 return false; 1095 } 1096 1097 // Prefer CL for zero-extended loads. 1098 if (CmpOp1.getOpcode() == ISD::ZERO_EXTEND || 1099 ISD::isZEXTLoad(CmpOp1.getNode())) 1100 return true; 1101 1102 // ...and for "in-register" zero extensions. 1103 if (CmpOp1.getOpcode() == ISD::AND && CmpOp1.getValueType() == MVT::i64) { 1104 SDValue Mask = CmpOp1.getOperand(1); 1105 if (Mask.getOpcode() == ISD::Constant && 1106 cast<ConstantSDNode>(Mask)->getZExtValue() == 0xffffffff) 1107 return true; 1108 } 1109 1110 return false; 1111} 1112 1113// Return a target node that compares CmpOp0 with CmpOp1 and stores a 1114// 2-bit result in CC. Set CCValid to the CCMASK_* of all possible 1115// 2-bit results and CCMask to the subset of those results that are 1116// associated with Cond. 1117static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 1118 ISD::CondCode Cond, unsigned &CCValid, 1119 unsigned &CCMask) { 1120 bool IsUnsigned = false; 1121 CCMask = CCMaskForCondCode(Cond); 1122 if (CmpOp0.getValueType().isFloatingPoint()) 1123 CCValid = SystemZ::CCMASK_FCMP; 1124 else { 1125 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 1126 CCValid = SystemZ::CCMASK_ICMP; 1127 CCMask &= CCValid; 1128 adjustZeroCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1129 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1130 if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask)) 1131 IsUnsigned = true; 1132 } 1133 1134 SDLoc DL(CmpOp0); 1135 return DAG.getNode((IsUnsigned ? SystemZISD::UCMP : SystemZISD::CMP), 1136 DL, MVT::Glue, CmpOp0, CmpOp1); 1137} 1138 1139// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1140// 64 bits. Extend is the extension type to use. Store the high part 1141// in Hi and the low part in Lo. 1142static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1143 unsigned Extend, SDValue Op0, SDValue Op1, 1144 SDValue &Hi, SDValue &Lo) { 1145 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1146 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1147 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1148 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1149 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1150 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1151} 1152 1153// Lower a binary operation that produces two VT results, one in each 1154// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1155// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1156// on the extended Op0 and (unextended) Op1. Store the even register result 1157// in Even and the odd register result in Odd. 1158static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1159 unsigned Extend, unsigned Opcode, 1160 SDValue Op0, SDValue Op1, 1161 SDValue &Even, SDValue &Odd) { 1162 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1163 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1164 SDValue(In128, 0), Op1); 1165 bool Is32Bit = is32Bit(VT); 1166 SDValue SubReg0 = DAG.getTargetConstant(SystemZ::even128(Is32Bit), VT); 1167 SDValue SubReg1 = DAG.getTargetConstant(SystemZ::odd128(Is32Bit), VT); 1168 SDNode *Reg0 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1169 VT, Result, SubReg0); 1170 SDNode *Reg1 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1171 VT, Result, SubReg1); 1172 Even = SDValue(Reg0, 0); 1173 Odd = SDValue(Reg1, 0); 1174} 1175 1176SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1177 SDValue Chain = Op.getOperand(0); 1178 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1179 SDValue CmpOp0 = Op.getOperand(2); 1180 SDValue CmpOp1 = Op.getOperand(3); 1181 SDValue Dest = Op.getOperand(4); 1182 SDLoc DL(Op); 1183 1184 unsigned CCValid, CCMask; 1185 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1186 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1187 Chain, DAG.getConstant(CCValid, MVT::i32), 1188 DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 1189} 1190 1191SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1192 SelectionDAG &DAG) const { 1193 SDValue CmpOp0 = Op.getOperand(0); 1194 SDValue CmpOp1 = Op.getOperand(1); 1195 SDValue TrueOp = Op.getOperand(2); 1196 SDValue FalseOp = Op.getOperand(3); 1197 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1198 SDLoc DL(Op); 1199 1200 unsigned CCValid, CCMask; 1201 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1202 1203 SmallVector<SDValue, 5> Ops; 1204 Ops.push_back(TrueOp); 1205 Ops.push_back(FalseOp); 1206 Ops.push_back(DAG.getConstant(CCValid, MVT::i32)); 1207 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1208 Ops.push_back(Flags); 1209 1210 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1211 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1212} 1213 1214SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1215 SelectionDAG &DAG) const { 1216 SDLoc DL(Node); 1217 const GlobalValue *GV = Node->getGlobal(); 1218 int64_t Offset = Node->getOffset(); 1219 EVT PtrVT = getPointerTy(); 1220 Reloc::Model RM = TM.getRelocationModel(); 1221 CodeModel::Model CM = TM.getCodeModel(); 1222 1223 SDValue Result; 1224 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1225 // Make sure that the offset is aligned to a halfword. If it isn't, 1226 // create an "anchor" at the previous 12-bit boundary. 1227 // FIXME check whether there is a better way of handling this. 1228 if (Offset & 1) { 1229 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 1230 Offset & ~uint64_t(0xfff)); 1231 Offset &= 0xfff; 1232 } else { 1233 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Offset); 1234 Offset = 0; 1235 } 1236 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1237 } else { 1238 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1239 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1240 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1241 MachinePointerInfo::getGOT(), false, false, false, 0); 1242 } 1243 1244 // If there was a non-zero offset that we didn't fold, create an explicit 1245 // addition for it. 1246 if (Offset != 0) 1247 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1248 DAG.getConstant(Offset, PtrVT)); 1249 1250 return Result; 1251} 1252 1253SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1254 SelectionDAG &DAG) const { 1255 SDLoc DL(Node); 1256 const GlobalValue *GV = Node->getGlobal(); 1257 EVT PtrVT = getPointerTy(); 1258 TLSModel::Model model = TM.getTLSModel(GV); 1259 1260 if (model != TLSModel::LocalExec) 1261 llvm_unreachable("only local-exec TLS mode supported"); 1262 1263 // The high part of the thread pointer is in access register 0. 1264 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1265 DAG.getConstant(0, MVT::i32)); 1266 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1267 1268 // The low part of the thread pointer is in access register 1. 1269 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1270 DAG.getConstant(1, MVT::i32)); 1271 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1272 1273 // Merge them into a single 64-bit address. 1274 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1275 DAG.getConstant(32, PtrVT)); 1276 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1277 1278 // Get the offset of GA from the thread pointer. 1279 SystemZConstantPoolValue *CPV = 1280 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1281 1282 // Force the offset into the constant pool and load it from there. 1283 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1284 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1285 CPAddr, MachinePointerInfo::getConstantPool(), 1286 false, false, false, 0); 1287 1288 // Add the base and offset together. 1289 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1290} 1291 1292SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1293 SelectionDAG &DAG) const { 1294 SDLoc DL(Node); 1295 const BlockAddress *BA = Node->getBlockAddress(); 1296 int64_t Offset = Node->getOffset(); 1297 EVT PtrVT = getPointerTy(); 1298 1299 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1300 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1301 return Result; 1302} 1303 1304SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1305 SelectionDAG &DAG) const { 1306 SDLoc DL(JT); 1307 EVT PtrVT = getPointerTy(); 1308 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1309 1310 // Use LARL to load the address of the table. 1311 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1312} 1313 1314SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1315 SelectionDAG &DAG) const { 1316 SDLoc DL(CP); 1317 EVT PtrVT = getPointerTy(); 1318 1319 SDValue Result; 1320 if (CP->isMachineConstantPoolEntry()) 1321 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1322 CP->getAlignment()); 1323 else 1324 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1325 CP->getAlignment(), CP->getOffset()); 1326 1327 // Use LARL to load the address of the constant pool entry. 1328 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1329} 1330 1331SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1332 SelectionDAG &DAG) const { 1333 SDLoc DL(Op); 1334 SDValue In = Op.getOperand(0); 1335 EVT InVT = In.getValueType(); 1336 EVT ResVT = Op.getValueType(); 1337 1338 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1339 SDValue Shift32 = DAG.getConstant(32, MVT::i64); 1340 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1341 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1342 SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); 1343 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); 1344 SDNode *Out = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1345 MVT::f32, Out64, SubReg32); 1346 return SDValue(Out, 0); 1347 } 1348 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1349 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1350 SDNode *In64 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1351 MVT::f64, SDValue(U64, 0), In, SubReg32); 1352 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, SDValue(In64, 0)); 1353 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); 1354 SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1355 return Out; 1356 } 1357 llvm_unreachable("Unexpected bitcast combination"); 1358} 1359 1360SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1361 SelectionDAG &DAG) const { 1362 MachineFunction &MF = DAG.getMachineFunction(); 1363 SystemZMachineFunctionInfo *FuncInfo = 1364 MF.getInfo<SystemZMachineFunctionInfo>(); 1365 EVT PtrVT = getPointerTy(); 1366 1367 SDValue Chain = Op.getOperand(0); 1368 SDValue Addr = Op.getOperand(1); 1369 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1370 SDLoc DL(Op); 1371 1372 // The initial values of each field. 1373 const unsigned NumFields = 4; 1374 SDValue Fields[NumFields] = { 1375 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1376 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1377 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1378 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1379 }; 1380 1381 // Store each field into its respective slot. 1382 SDValue MemOps[NumFields]; 1383 unsigned Offset = 0; 1384 for (unsigned I = 0; I < NumFields; ++I) { 1385 SDValue FieldAddr = Addr; 1386 if (Offset != 0) 1387 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1388 DAG.getIntPtrConstant(Offset)); 1389 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1390 MachinePointerInfo(SV, Offset), 1391 false, false, 0); 1392 Offset += 8; 1393 } 1394 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1395} 1396 1397SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1398 SelectionDAG &DAG) const { 1399 SDValue Chain = Op.getOperand(0); 1400 SDValue DstPtr = Op.getOperand(1); 1401 SDValue SrcPtr = Op.getOperand(2); 1402 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1403 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1404 SDLoc DL(Op); 1405 1406 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1407 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1408 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1409} 1410 1411SDValue SystemZTargetLowering:: 1412lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1413 SDValue Chain = Op.getOperand(0); 1414 SDValue Size = Op.getOperand(1); 1415 SDLoc DL(Op); 1416 1417 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1418 1419 // Get a reference to the stack pointer. 1420 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1421 1422 // Get the new stack pointer value. 1423 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1424 1425 // Copy the new stack pointer back. 1426 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1427 1428 // The allocated data lives above the 160 bytes allocated for the standard 1429 // frame, plus any outgoing stack arguments. We don't know how much that 1430 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1431 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1432 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1433 1434 SDValue Ops[2] = { Result, Chain }; 1435 return DAG.getMergeValues(Ops, 2, DL); 1436} 1437 1438SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 1439 SelectionDAG &DAG) const { 1440 EVT VT = Op.getValueType(); 1441 SDLoc DL(Op); 1442 SDValue Ops[2]; 1443 if (is32Bit(VT)) 1444 // Just do a normal 64-bit multiplication and extract the results. 1445 // We define this so that it can be used for constant division. 1446 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 1447 Op.getOperand(1), Ops[1], Ops[0]); 1448 else { 1449 // Do a full 128-bit multiplication based on UMUL_LOHI64: 1450 // 1451 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 1452 // 1453 // but using the fact that the upper halves are either all zeros 1454 // or all ones: 1455 // 1456 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 1457 // 1458 // and grouping the right terms together since they are quicker than the 1459 // multiplication: 1460 // 1461 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 1462 SDValue C63 = DAG.getConstant(63, MVT::i64); 1463 SDValue LL = Op.getOperand(0); 1464 SDValue RL = Op.getOperand(1); 1465 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 1466 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 1467 // UMUL_LOHI64 returns the low result in the odd register and the high 1468 // result in the even register. SMUL_LOHI is defined to return the 1469 // low half first, so the results are in reverse order. 1470 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1471 LL, RL, Ops[1], Ops[0]); 1472 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 1473 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 1474 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 1475 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 1476 } 1477 return DAG.getMergeValues(Ops, 2, DL); 1478} 1479 1480SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1481 SelectionDAG &DAG) const { 1482 EVT VT = Op.getValueType(); 1483 SDLoc DL(Op); 1484 SDValue Ops[2]; 1485 if (is32Bit(VT)) 1486 // Just do a normal 64-bit multiplication and extract the results. 1487 // We define this so that it can be used for constant division. 1488 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 1489 Op.getOperand(1), Ops[1], Ops[0]); 1490 else 1491 // UMUL_LOHI64 returns the low result in the odd register and the high 1492 // result in the even register. UMUL_LOHI is defined to return the 1493 // low half first, so the results are in reverse order. 1494 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1495 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1496 return DAG.getMergeValues(Ops, 2, DL); 1497} 1498 1499SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1500 SelectionDAG &DAG) const { 1501 SDValue Op0 = Op.getOperand(0); 1502 SDValue Op1 = Op.getOperand(1); 1503 EVT VT = Op.getValueType(); 1504 SDLoc DL(Op); 1505 unsigned Opcode; 1506 1507 // We use DSGF for 32-bit division. 1508 if (is32Bit(VT)) { 1509 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1510 Opcode = SystemZISD::SDIVREM32; 1511 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 1512 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 1513 Opcode = SystemZISD::SDIVREM32; 1514 } else 1515 Opcode = SystemZISD::SDIVREM64; 1516 1517 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1518 // input is "don't care". The instruction returns the remainder in 1519 // the even register and the quotient in the odd register. 1520 SDValue Ops[2]; 1521 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 1522 Op0, Op1, Ops[1], Ops[0]); 1523 return DAG.getMergeValues(Ops, 2, DL); 1524} 1525 1526SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1527 SelectionDAG &DAG) const { 1528 EVT VT = Op.getValueType(); 1529 SDLoc DL(Op); 1530 1531 // DL(G) uses a double-width dividend, so we need to clear the even 1532 // register in the GR128 input. The instruction returns the remainder 1533 // in the even register and the quotient in the odd register. 1534 SDValue Ops[2]; 1535 if (is32Bit(VT)) 1536 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1537 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1538 else 1539 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1540 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1541 return DAG.getMergeValues(Ops, 2, DL); 1542} 1543 1544SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1545 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1546 1547 // Get the known-zero masks for each operand. 1548 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1549 APInt KnownZero[2], KnownOne[2]; 1550 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1551 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1552 1553 // See if the upper 32 bits of one operand and the lower 32 bits of the 1554 // other are known zero. They are the low and high operands respectively. 1555 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1556 KnownZero[1].getZExtValue() }; 1557 unsigned High, Low; 1558 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1559 High = 1, Low = 0; 1560 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1561 High = 0, Low = 1; 1562 else 1563 return Op; 1564 1565 SDValue LowOp = Ops[Low]; 1566 SDValue HighOp = Ops[High]; 1567 1568 // If the high part is a constant, we're better off using IILH. 1569 if (HighOp.getOpcode() == ISD::Constant) 1570 return Op; 1571 1572 // If the low part is a constant that is outside the range of LHI, 1573 // then we're better off using IILF. 1574 if (LowOp.getOpcode() == ISD::Constant) { 1575 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1576 if (!isInt<16>(Value)) 1577 return Op; 1578 } 1579 1580 // Check whether the high part is an AND that doesn't change the 1581 // high 32 bits and just masks out low bits. We can skip it if so. 1582 if (HighOp.getOpcode() == ISD::AND && 1583 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1584 ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); 1585 uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; 1586 if ((Mask >> 32) == 0xffffffff) 1587 HighOp = HighOp.getOperand(0); 1588 } 1589 1590 // Take advantage of the fact that all GR32 operations only change the 1591 // low 32 bits by truncating Low to an i32 and inserting it directly 1592 // using a subreg. The interesting cases are those where the truncation 1593 // can be folded. 1594 SDLoc DL(Op); 1595 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1596 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1597 SDNode *Result = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1598 MVT::i64, HighOp, Low32, SubReg32); 1599 return SDValue(Result, 0); 1600} 1601 1602// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1603// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1604SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1605 SelectionDAG &DAG, 1606 unsigned Opcode) const { 1607 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1608 1609 // 32-bit operations need no code outside the main loop. 1610 EVT NarrowVT = Node->getMemoryVT(); 1611 EVT WideVT = MVT::i32; 1612 if (NarrowVT == WideVT) 1613 return Op; 1614 1615 int64_t BitSize = NarrowVT.getSizeInBits(); 1616 SDValue ChainIn = Node->getChain(); 1617 SDValue Addr = Node->getBasePtr(); 1618 SDValue Src2 = Node->getVal(); 1619 MachineMemOperand *MMO = Node->getMemOperand(); 1620 SDLoc DL(Node); 1621 EVT PtrVT = Addr.getValueType(); 1622 1623 // Convert atomic subtracts of constants into additions. 1624 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1625 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1626 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1627 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1628 } 1629 1630 // Get the address of the containing word. 1631 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1632 DAG.getConstant(-4, PtrVT)); 1633 1634 // Get the number of bits that the word must be rotated left in order 1635 // to bring the field to the top bits of a GR32. 1636 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1637 DAG.getConstant(3, PtrVT)); 1638 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1639 1640 // Get the complementing shift amount, for rotating a field in the top 1641 // bits back to its proper position. 1642 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1643 DAG.getConstant(0, WideVT), BitShift); 1644 1645 // Extend the source operand to 32 bits and prepare it for the inner loop. 1646 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1647 // operations require the source to be shifted in advance. (This shift 1648 // can be folded if the source is constant.) For AND and NAND, the lower 1649 // bits must be set, while for other opcodes they should be left clear. 1650 if (Opcode != SystemZISD::ATOMIC_SWAPW) 1651 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 1652 DAG.getConstant(32 - BitSize, WideVT)); 1653 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 1654 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 1655 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 1656 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 1657 1658 // Construct the ATOMIC_LOADW_* node. 1659 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1660 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 1661 DAG.getConstant(BitSize, WideVT) }; 1662 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 1663 array_lengthof(Ops), 1664 NarrowVT, MMO); 1665 1666 // Rotate the result of the final CS so that the field is in the lower 1667 // bits of a GR32, then truncate it. 1668 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 1669 DAG.getConstant(BitSize, WideVT)); 1670 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 1671 1672 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 1673 return DAG.getMergeValues(RetOps, 2, DL); 1674} 1675 1676// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 1677// into a fullword ATOMIC_CMP_SWAPW operation. 1678SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 1679 SelectionDAG &DAG) const { 1680 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1681 1682 // We have native support for 32-bit compare and swap. 1683 EVT NarrowVT = Node->getMemoryVT(); 1684 EVT WideVT = MVT::i32; 1685 if (NarrowVT == WideVT) 1686 return Op; 1687 1688 int64_t BitSize = NarrowVT.getSizeInBits(); 1689 SDValue ChainIn = Node->getOperand(0); 1690 SDValue Addr = Node->getOperand(1); 1691 SDValue CmpVal = Node->getOperand(2); 1692 SDValue SwapVal = Node->getOperand(3); 1693 MachineMemOperand *MMO = Node->getMemOperand(); 1694 SDLoc DL(Node); 1695 EVT PtrVT = Addr.getValueType(); 1696 1697 // Get the address of the containing word. 1698 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1699 DAG.getConstant(-4, PtrVT)); 1700 1701 // Get the number of bits that the word must be rotated left in order 1702 // to bring the field to the top bits of a GR32. 1703 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1704 DAG.getConstant(3, PtrVT)); 1705 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1706 1707 // Get the complementing shift amount, for rotating a field in the top 1708 // bits back to its proper position. 1709 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1710 DAG.getConstant(0, WideVT), BitShift); 1711 1712 // Construct the ATOMIC_CMP_SWAPW node. 1713 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1714 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 1715 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 1716 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 1717 VTList, Ops, array_lengthof(Ops), 1718 NarrowVT, MMO); 1719 return AtomicOp; 1720} 1721 1722SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 1723 SelectionDAG &DAG) const { 1724 MachineFunction &MF = DAG.getMachineFunction(); 1725 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1726 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 1727 SystemZ::R15D, Op.getValueType()); 1728} 1729 1730SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 1731 SelectionDAG &DAG) const { 1732 MachineFunction &MF = DAG.getMachineFunction(); 1733 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1734 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 1735 SystemZ::R15D, Op.getOperand(1)); 1736} 1737 1738SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 1739 SelectionDAG &DAG) const { 1740 switch (Op.getOpcode()) { 1741 case ISD::BR_CC: 1742 return lowerBR_CC(Op, DAG); 1743 case ISD::SELECT_CC: 1744 return lowerSELECT_CC(Op, DAG); 1745 case ISD::GlobalAddress: 1746 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 1747 case ISD::GlobalTLSAddress: 1748 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 1749 case ISD::BlockAddress: 1750 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 1751 case ISD::JumpTable: 1752 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 1753 case ISD::ConstantPool: 1754 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 1755 case ISD::BITCAST: 1756 return lowerBITCAST(Op, DAG); 1757 case ISD::VASTART: 1758 return lowerVASTART(Op, DAG); 1759 case ISD::VACOPY: 1760 return lowerVACOPY(Op, DAG); 1761 case ISD::DYNAMIC_STACKALLOC: 1762 return lowerDYNAMIC_STACKALLOC(Op, DAG); 1763 case ISD::SMUL_LOHI: 1764 return lowerSMUL_LOHI(Op, DAG); 1765 case ISD::UMUL_LOHI: 1766 return lowerUMUL_LOHI(Op, DAG); 1767 case ISD::SDIVREM: 1768 return lowerSDIVREM(Op, DAG); 1769 case ISD::UDIVREM: 1770 return lowerUDIVREM(Op, DAG); 1771 case ISD::OR: 1772 return lowerOR(Op, DAG); 1773 case ISD::ATOMIC_SWAP: 1774 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 1775 case ISD::ATOMIC_LOAD_ADD: 1776 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 1777 case ISD::ATOMIC_LOAD_SUB: 1778 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 1779 case ISD::ATOMIC_LOAD_AND: 1780 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 1781 case ISD::ATOMIC_LOAD_OR: 1782 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 1783 case ISD::ATOMIC_LOAD_XOR: 1784 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 1785 case ISD::ATOMIC_LOAD_NAND: 1786 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 1787 case ISD::ATOMIC_LOAD_MIN: 1788 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 1789 case ISD::ATOMIC_LOAD_MAX: 1790 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 1791 case ISD::ATOMIC_LOAD_UMIN: 1792 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 1793 case ISD::ATOMIC_LOAD_UMAX: 1794 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 1795 case ISD::ATOMIC_CMP_SWAP: 1796 return lowerATOMIC_CMP_SWAP(Op, DAG); 1797 case ISD::STACKSAVE: 1798 return lowerSTACKSAVE(Op, DAG); 1799 case ISD::STACKRESTORE: 1800 return lowerSTACKRESTORE(Op, DAG); 1801 default: 1802 llvm_unreachable("Unexpected node to lower"); 1803 } 1804} 1805 1806const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 1807#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 1808 switch (Opcode) { 1809 OPCODE(RET_FLAG); 1810 OPCODE(CALL); 1811 OPCODE(SIBCALL); 1812 OPCODE(PCREL_WRAPPER); 1813 OPCODE(CMP); 1814 OPCODE(UCMP); 1815 OPCODE(BR_CCMASK); 1816 OPCODE(SELECT_CCMASK); 1817 OPCODE(ADJDYNALLOC); 1818 OPCODE(EXTRACT_ACCESS); 1819 OPCODE(UMUL_LOHI64); 1820 OPCODE(SDIVREM64); 1821 OPCODE(UDIVREM32); 1822 OPCODE(UDIVREM64); 1823 OPCODE(MVC); 1824 OPCODE(CLC); 1825 OPCODE(STRCMP); 1826 OPCODE(STPCPY); 1827 OPCODE(SEARCH_STRING); 1828 OPCODE(IPM); 1829 OPCODE(ATOMIC_SWAPW); 1830 OPCODE(ATOMIC_LOADW_ADD); 1831 OPCODE(ATOMIC_LOADW_SUB); 1832 OPCODE(ATOMIC_LOADW_AND); 1833 OPCODE(ATOMIC_LOADW_OR); 1834 OPCODE(ATOMIC_LOADW_XOR); 1835 OPCODE(ATOMIC_LOADW_NAND); 1836 OPCODE(ATOMIC_LOADW_MIN); 1837 OPCODE(ATOMIC_LOADW_MAX); 1838 OPCODE(ATOMIC_LOADW_UMIN); 1839 OPCODE(ATOMIC_LOADW_UMAX); 1840 OPCODE(ATOMIC_CMP_SWAPW); 1841 } 1842 return NULL; 1843#undef OPCODE 1844} 1845 1846//===----------------------------------------------------------------------===// 1847// Custom insertion 1848//===----------------------------------------------------------------------===// 1849 1850// Create a new basic block after MBB. 1851static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 1852 MachineFunction &MF = *MBB->getParent(); 1853 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1854 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 1855 return NewMBB; 1856} 1857 1858// Split MBB after MI and return the new block (the one that contains 1859// instructions after MI). 1860static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 1861 MachineBasicBlock *MBB) { 1862 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1863 NewMBB->splice(NewMBB->begin(), MBB, 1864 llvm::next(MachineBasicBlock::iterator(MI)), 1865 MBB->end()); 1866 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1867 return NewMBB; 1868} 1869 1870// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 1871MachineBasicBlock * 1872SystemZTargetLowering::emitSelect(MachineInstr *MI, 1873 MachineBasicBlock *MBB) const { 1874 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1875 1876 unsigned DestReg = MI->getOperand(0).getReg(); 1877 unsigned TrueReg = MI->getOperand(1).getReg(); 1878 unsigned FalseReg = MI->getOperand(2).getReg(); 1879 unsigned CCValid = MI->getOperand(3).getImm(); 1880 unsigned CCMask = MI->getOperand(4).getImm(); 1881 DebugLoc DL = MI->getDebugLoc(); 1882 1883 MachineBasicBlock *StartMBB = MBB; 1884 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 1885 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 1886 1887 // StartMBB: 1888 // BRC CCMask, JoinMBB 1889 // # fallthrough to FalseMBB 1890 MBB = StartMBB; 1891 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 1892 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 1893 MBB->addSuccessor(JoinMBB); 1894 MBB->addSuccessor(FalseMBB); 1895 1896 // FalseMBB: 1897 // # fallthrough to JoinMBB 1898 MBB = FalseMBB; 1899 MBB->addSuccessor(JoinMBB); 1900 1901 // JoinMBB: 1902 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 1903 // ... 1904 MBB = JoinMBB; 1905 BuildMI(*MBB, MBB->begin(), DL, TII->get(SystemZ::PHI), DestReg) 1906 .addReg(TrueReg).addMBB(StartMBB) 1907 .addReg(FalseReg).addMBB(FalseMBB); 1908 1909 MI->eraseFromParent(); 1910 return JoinMBB; 1911} 1912 1913// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 1914// StoreOpcode is the store to use and Invert says whether the store should 1915// happen when the condition is false rather than true. If a STORE ON 1916// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 1917MachineBasicBlock * 1918SystemZTargetLowering::emitCondStore(MachineInstr *MI, 1919 MachineBasicBlock *MBB, 1920 unsigned StoreOpcode, unsigned STOCOpcode, 1921 bool Invert) const { 1922 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1923 1924 unsigned SrcReg = MI->getOperand(0).getReg(); 1925 MachineOperand Base = MI->getOperand(1); 1926 int64_t Disp = MI->getOperand(2).getImm(); 1927 unsigned IndexReg = MI->getOperand(3).getReg(); 1928 unsigned CCValid = MI->getOperand(4).getImm(); 1929 unsigned CCMask = MI->getOperand(5).getImm(); 1930 DebugLoc DL = MI->getDebugLoc(); 1931 1932 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 1933 1934 // Use STOCOpcode if possible. We could use different store patterns in 1935 // order to avoid matching the index register, but the performance trade-offs 1936 // might be more complicated in that case. 1937 if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) { 1938 if (Invert) 1939 CCMask ^= CCValid; 1940 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 1941 .addReg(SrcReg).addOperand(Base).addImm(Disp) 1942 .addImm(CCValid).addImm(CCMask); 1943 MI->eraseFromParent(); 1944 return MBB; 1945 } 1946 1947 // Get the condition needed to branch around the store. 1948 if (!Invert) 1949 CCMask ^= CCValid; 1950 1951 MachineBasicBlock *StartMBB = MBB; 1952 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 1953 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 1954 1955 // StartMBB: 1956 // BRC CCMask, JoinMBB 1957 // # fallthrough to FalseMBB 1958 MBB = StartMBB; 1959 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 1960 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 1961 MBB->addSuccessor(JoinMBB); 1962 MBB->addSuccessor(FalseMBB); 1963 1964 // FalseMBB: 1965 // store %SrcReg, %Disp(%Index,%Base) 1966 // # fallthrough to JoinMBB 1967 MBB = FalseMBB; 1968 BuildMI(MBB, DL, TII->get(StoreOpcode)) 1969 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 1970 MBB->addSuccessor(JoinMBB); 1971 1972 MI->eraseFromParent(); 1973 return JoinMBB; 1974} 1975 1976// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 1977// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 1978// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 1979// BitSize is the width of the field in bits, or 0 if this is a partword 1980// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 1981// is one of the operands. Invert says whether the field should be 1982// inverted after performing BinOpcode (e.g. for NAND). 1983MachineBasicBlock * 1984SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 1985 MachineBasicBlock *MBB, 1986 unsigned BinOpcode, 1987 unsigned BitSize, 1988 bool Invert) const { 1989 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1990 MachineFunction &MF = *MBB->getParent(); 1991 MachineRegisterInfo &MRI = MF.getRegInfo(); 1992 bool IsSubWord = (BitSize < 32); 1993 1994 // Extract the operands. Base can be a register or a frame index. 1995 // Src2 can be a register or immediate. 1996 unsigned Dest = MI->getOperand(0).getReg(); 1997 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1998 int64_t Disp = MI->getOperand(2).getImm(); 1999 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2000 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2001 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2002 DebugLoc DL = MI->getDebugLoc(); 2003 if (IsSubWord) 2004 BitSize = MI->getOperand(6).getImm(); 2005 2006 // Subword operations use 32-bit registers. 2007 const TargetRegisterClass *RC = (BitSize <= 32 ? 2008 &SystemZ::GR32BitRegClass : 2009 &SystemZ::GR64BitRegClass); 2010 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2011 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2012 2013 // Get the right opcodes for the displacement. 2014 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2015 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2016 assert(LOpcode && CSOpcode && "Displacement out of range"); 2017 2018 // Create virtual registers for temporary results. 2019 unsigned OrigVal = MRI.createVirtualRegister(RC); 2020 unsigned OldVal = MRI.createVirtualRegister(RC); 2021 unsigned NewVal = (BinOpcode || IsSubWord ? 2022 MRI.createVirtualRegister(RC) : Src2.getReg()); 2023 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2024 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2025 2026 // Insert a basic block for the main loop. 2027 MachineBasicBlock *StartMBB = MBB; 2028 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2029 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2030 2031 // StartMBB: 2032 // ... 2033 // %OrigVal = L Disp(%Base) 2034 // # fall through to LoopMMB 2035 MBB = StartMBB; 2036 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2037 .addOperand(Base).addImm(Disp).addReg(0); 2038 MBB->addSuccessor(LoopMBB); 2039 2040 // LoopMBB: 2041 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2042 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2043 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2044 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2045 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2046 // JNE LoopMBB 2047 // # fall through to DoneMMB 2048 MBB = LoopMBB; 2049 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2050 .addReg(OrigVal).addMBB(StartMBB) 2051 .addReg(Dest).addMBB(LoopMBB); 2052 if (IsSubWord) 2053 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2054 .addReg(OldVal).addReg(BitShift).addImm(0); 2055 if (Invert) { 2056 // Perform the operation normally and then invert every bit of the field. 2057 unsigned Tmp = MRI.createVirtualRegister(RC); 2058 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2059 .addReg(RotatedOldVal).addOperand(Src2); 2060 if (BitSize < 32) 2061 // XILF with the upper BitSize bits set. 2062 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 2063 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 2064 else if (BitSize == 32) 2065 // XILF with every bit set. 2066 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 2067 .addReg(Tmp).addImm(~uint32_t(0)); 2068 else { 2069 // Use LCGR and add -1 to the result, which is more compact than 2070 // an XILF, XILH pair. 2071 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2072 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2073 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2074 .addReg(Tmp2).addImm(-1); 2075 } 2076 } else if (BinOpcode) 2077 // A simply binary operation. 2078 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2079 .addReg(RotatedOldVal).addOperand(Src2); 2080 else if (IsSubWord) 2081 // Use RISBG to rotate Src2 into position and use it to replace the 2082 // field in RotatedOldVal. 2083 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2084 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2085 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2086 if (IsSubWord) 2087 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2088 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2089 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2090 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2091 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2092 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2093 MBB->addSuccessor(LoopMBB); 2094 MBB->addSuccessor(DoneMBB); 2095 2096 MI->eraseFromParent(); 2097 return DoneMBB; 2098} 2099 2100// Implement EmitInstrWithCustomInserter for pseudo 2101// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2102// instruction that should be used to compare the current field with the 2103// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2104// for when the current field should be kept. BitSize is the width of 2105// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2106MachineBasicBlock * 2107SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2108 MachineBasicBlock *MBB, 2109 unsigned CompareOpcode, 2110 unsigned KeepOldMask, 2111 unsigned BitSize) const { 2112 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2113 MachineFunction &MF = *MBB->getParent(); 2114 MachineRegisterInfo &MRI = MF.getRegInfo(); 2115 bool IsSubWord = (BitSize < 32); 2116 2117 // Extract the operands. Base can be a register or a frame index. 2118 unsigned Dest = MI->getOperand(0).getReg(); 2119 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2120 int64_t Disp = MI->getOperand(2).getImm(); 2121 unsigned Src2 = MI->getOperand(3).getReg(); 2122 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2123 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2124 DebugLoc DL = MI->getDebugLoc(); 2125 if (IsSubWord) 2126 BitSize = MI->getOperand(6).getImm(); 2127 2128 // Subword operations use 32-bit registers. 2129 const TargetRegisterClass *RC = (BitSize <= 32 ? 2130 &SystemZ::GR32BitRegClass : 2131 &SystemZ::GR64BitRegClass); 2132 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2133 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2134 2135 // Get the right opcodes for the displacement. 2136 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2137 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2138 assert(LOpcode && CSOpcode && "Displacement out of range"); 2139 2140 // Create virtual registers for temporary results. 2141 unsigned OrigVal = MRI.createVirtualRegister(RC); 2142 unsigned OldVal = MRI.createVirtualRegister(RC); 2143 unsigned NewVal = MRI.createVirtualRegister(RC); 2144 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2145 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2146 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2147 2148 // Insert 3 basic blocks for the loop. 2149 MachineBasicBlock *StartMBB = MBB; 2150 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2151 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2152 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2153 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2154 2155 // StartMBB: 2156 // ... 2157 // %OrigVal = L Disp(%Base) 2158 // # fall through to LoopMMB 2159 MBB = StartMBB; 2160 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2161 .addOperand(Base).addImm(Disp).addReg(0); 2162 MBB->addSuccessor(LoopMBB); 2163 2164 // LoopMBB: 2165 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2166 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2167 // CompareOpcode %RotatedOldVal, %Src2 2168 // BRC KeepOldMask, UpdateMBB 2169 MBB = LoopMBB; 2170 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2171 .addReg(OrigVal).addMBB(StartMBB) 2172 .addReg(Dest).addMBB(UpdateMBB); 2173 if (IsSubWord) 2174 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2175 .addReg(OldVal).addReg(BitShift).addImm(0); 2176 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2177 .addReg(RotatedOldVal).addReg(Src2); 2178 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2179 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2180 MBB->addSuccessor(UpdateMBB); 2181 MBB->addSuccessor(UseAltMBB); 2182 2183 // UseAltMBB: 2184 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2185 // # fall through to UpdateMMB 2186 MBB = UseAltMBB; 2187 if (IsSubWord) 2188 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2189 .addReg(RotatedOldVal).addReg(Src2) 2190 .addImm(32).addImm(31 + BitSize).addImm(0); 2191 MBB->addSuccessor(UpdateMBB); 2192 2193 // UpdateMBB: 2194 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2195 // [ %RotatedAltVal, UseAltMBB ] 2196 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2197 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2198 // JNE LoopMBB 2199 // # fall through to DoneMMB 2200 MBB = UpdateMBB; 2201 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2202 .addReg(RotatedOldVal).addMBB(LoopMBB) 2203 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2204 if (IsSubWord) 2205 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2206 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2207 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2208 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2209 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2210 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2211 MBB->addSuccessor(LoopMBB); 2212 MBB->addSuccessor(DoneMBB); 2213 2214 MI->eraseFromParent(); 2215 return DoneMBB; 2216} 2217 2218// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2219// instruction MI. 2220MachineBasicBlock * 2221SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2222 MachineBasicBlock *MBB) const { 2223 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2224 MachineFunction &MF = *MBB->getParent(); 2225 MachineRegisterInfo &MRI = MF.getRegInfo(); 2226 2227 // Extract the operands. Base can be a register or a frame index. 2228 unsigned Dest = MI->getOperand(0).getReg(); 2229 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2230 int64_t Disp = MI->getOperand(2).getImm(); 2231 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2232 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2233 unsigned BitShift = MI->getOperand(5).getReg(); 2234 unsigned NegBitShift = MI->getOperand(6).getReg(); 2235 int64_t BitSize = MI->getOperand(7).getImm(); 2236 DebugLoc DL = MI->getDebugLoc(); 2237 2238 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2239 2240 // Get the right opcodes for the displacement. 2241 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2242 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2243 assert(LOpcode && CSOpcode && "Displacement out of range"); 2244 2245 // Create virtual registers for temporary results. 2246 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2247 unsigned OldVal = MRI.createVirtualRegister(RC); 2248 unsigned CmpVal = MRI.createVirtualRegister(RC); 2249 unsigned SwapVal = MRI.createVirtualRegister(RC); 2250 unsigned StoreVal = MRI.createVirtualRegister(RC); 2251 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2252 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2253 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2254 2255 // Insert 2 basic blocks for the loop. 2256 MachineBasicBlock *StartMBB = MBB; 2257 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2258 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2259 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 2260 2261 // StartMBB: 2262 // ... 2263 // %OrigOldVal = L Disp(%Base) 2264 // # fall through to LoopMMB 2265 MBB = StartMBB; 2266 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 2267 .addOperand(Base).addImm(Disp).addReg(0); 2268 MBB->addSuccessor(LoopMBB); 2269 2270 // LoopMBB: 2271 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 2272 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 2273 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 2274 // %Dest = RLL %OldVal, BitSize(%BitShift) 2275 // ^^ The low BitSize bits contain the field 2276 // of interest. 2277 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 2278 // ^^ Replace the upper 32-BitSize bits of the 2279 // comparison value with those that we loaded, 2280 // so that we can use a full word comparison. 2281 // CR %Dest, %RetryCmpVal 2282 // JNE DoneMBB 2283 // # Fall through to SetMBB 2284 MBB = LoopMBB; 2285 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2286 .addReg(OrigOldVal).addMBB(StartMBB) 2287 .addReg(RetryOldVal).addMBB(SetMBB); 2288 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 2289 .addReg(OrigCmpVal).addMBB(StartMBB) 2290 .addReg(RetryCmpVal).addMBB(SetMBB); 2291 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 2292 .addReg(OrigSwapVal).addMBB(StartMBB) 2293 .addReg(RetrySwapVal).addMBB(SetMBB); 2294 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 2295 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 2296 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 2297 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2298 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 2299 .addReg(Dest).addReg(RetryCmpVal); 2300 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2301 .addImm(SystemZ::CCMASK_ICMP) 2302 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 2303 MBB->addSuccessor(DoneMBB); 2304 MBB->addSuccessor(SetMBB); 2305 2306 // SetMBB: 2307 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 2308 // ^^ Replace the upper 32-BitSize bits of the new 2309 // value with those that we loaded. 2310 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 2311 // ^^ Rotate the new field to its proper position. 2312 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 2313 // JNE LoopMBB 2314 // # fall through to ExitMMB 2315 MBB = SetMBB; 2316 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 2317 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2318 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 2319 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 2320 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2321 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2322 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2323 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2324 MBB->addSuccessor(LoopMBB); 2325 MBB->addSuccessor(DoneMBB); 2326 2327 MI->eraseFromParent(); 2328 return DoneMBB; 2329} 2330 2331// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2332// if the high register of the GR128 value must be cleared or false if 2333// it's "don't care". SubReg is subreg_odd32 when extending a GR32 2334// and subreg_odd when extending a GR64. 2335MachineBasicBlock * 2336SystemZTargetLowering::emitExt128(MachineInstr *MI, 2337 MachineBasicBlock *MBB, 2338 bool ClearEven, unsigned SubReg) const { 2339 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2340 MachineFunction &MF = *MBB->getParent(); 2341 MachineRegisterInfo &MRI = MF.getRegInfo(); 2342 DebugLoc DL = MI->getDebugLoc(); 2343 2344 unsigned Dest = MI->getOperand(0).getReg(); 2345 unsigned Src = MI->getOperand(1).getReg(); 2346 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2347 2348 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2349 if (ClearEven) { 2350 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2351 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2352 2353 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2354 .addImm(0); 2355 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2356 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_high); 2357 In128 = NewIn128; 2358 } 2359 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2360 .addReg(In128).addReg(Src).addImm(SubReg); 2361 2362 MI->eraseFromParent(); 2363 return MBB; 2364} 2365 2366MachineBasicBlock * 2367SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 2368 MachineBasicBlock *MBB, 2369 unsigned Opcode) const { 2370 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2371 DebugLoc DL = MI->getDebugLoc(); 2372 2373 MachineOperand DestBase = MI->getOperand(0); 2374 uint64_t DestDisp = MI->getOperand(1).getImm(); 2375 MachineOperand SrcBase = MI->getOperand(2); 2376 uint64_t SrcDisp = MI->getOperand(3).getImm(); 2377 uint64_t Length = MI->getOperand(4).getImm(); 2378 2379 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 2380 .addOperand(DestBase).addImm(DestDisp).addImm(Length) 2381 .addOperand(SrcBase).addImm(SrcDisp); 2382 2383 MI->eraseFromParent(); 2384 return MBB; 2385} 2386 2387// Decompose string pseudo-instruction MI into a loop that continually performs 2388// Opcode until CC != 3. 2389MachineBasicBlock * 2390SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 2391 MachineBasicBlock *MBB, 2392 unsigned Opcode) const { 2393 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2394 MachineFunction &MF = *MBB->getParent(); 2395 MachineRegisterInfo &MRI = MF.getRegInfo(); 2396 DebugLoc DL = MI->getDebugLoc(); 2397 2398 uint64_t End1Reg = MI->getOperand(0).getReg(); 2399 uint64_t Start1Reg = MI->getOperand(1).getReg(); 2400 uint64_t Start2Reg = MI->getOperand(2).getReg(); 2401 uint64_t CharReg = MI->getOperand(3).getReg(); 2402 2403 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 2404 uint64_t This1Reg = MRI.createVirtualRegister(RC); 2405 uint64_t This2Reg = MRI.createVirtualRegister(RC); 2406 uint64_t End2Reg = MRI.createVirtualRegister(RC); 2407 2408 MachineBasicBlock *StartMBB = MBB; 2409 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 2410 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2411 2412 // StartMBB: 2413 // # fall through to LoopMMB 2414 MBB->addSuccessor(LoopMBB); 2415 2416 // LoopMBB: 2417 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 2418 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 2419 // R0W = %CharReg 2420 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0W 2421 // JO LoopMBB 2422 // # fall through to DoneMMB 2423 // 2424 // The load of R0W can be hoisted by post-RA LICM. 2425 MBB = LoopMBB; 2426 2427 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 2428 .addReg(Start1Reg).addMBB(StartMBB) 2429 .addReg(End1Reg).addMBB(LoopMBB); 2430 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 2431 .addReg(Start2Reg).addMBB(StartMBB) 2432 .addReg(End2Reg).addMBB(LoopMBB); 2433 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0W).addReg(CharReg); 2434 BuildMI(MBB, DL, TII->get(Opcode)) 2435 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 2436 .addReg(This1Reg).addReg(This2Reg); 2437 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2438 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 2439 MBB->addSuccessor(LoopMBB); 2440 MBB->addSuccessor(DoneMBB); 2441 2442 DoneMBB->addLiveIn(SystemZ::CC); 2443 2444 MI->eraseFromParent(); 2445 return DoneMBB; 2446} 2447 2448MachineBasicBlock *SystemZTargetLowering:: 2449EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 2450 switch (MI->getOpcode()) { 2451 case SystemZ::Select32: 2452 case SystemZ::SelectF32: 2453 case SystemZ::Select64: 2454 case SystemZ::SelectF64: 2455 case SystemZ::SelectF128: 2456 return emitSelect(MI, MBB); 2457 2458 case SystemZ::CondStore8_32: 2459 return emitCondStore(MI, MBB, SystemZ::STC32, 0, false); 2460 case SystemZ::CondStore8_32Inv: 2461 return emitCondStore(MI, MBB, SystemZ::STC32, 0, true); 2462 case SystemZ::CondStore16_32: 2463 return emitCondStore(MI, MBB, SystemZ::STH32, 0, false); 2464 case SystemZ::CondStore16_32Inv: 2465 return emitCondStore(MI, MBB, SystemZ::STH32, 0, true); 2466 case SystemZ::CondStore32_32: 2467 return emitCondStore(MI, MBB, SystemZ::ST32, SystemZ::STOC32, false); 2468 case SystemZ::CondStore32_32Inv: 2469 return emitCondStore(MI, MBB, SystemZ::ST32, SystemZ::STOC32, true); 2470 case SystemZ::CondStore8: 2471 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 2472 case SystemZ::CondStore8Inv: 2473 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 2474 case SystemZ::CondStore16: 2475 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 2476 case SystemZ::CondStore16Inv: 2477 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 2478 case SystemZ::CondStore32: 2479 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 2480 case SystemZ::CondStore32Inv: 2481 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 2482 case SystemZ::CondStore64: 2483 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 2484 case SystemZ::CondStore64Inv: 2485 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 2486 case SystemZ::CondStoreF32: 2487 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 2488 case SystemZ::CondStoreF32Inv: 2489 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 2490 case SystemZ::CondStoreF64: 2491 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 2492 case SystemZ::CondStoreF64Inv: 2493 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 2494 2495 case SystemZ::AEXT128_64: 2496 return emitExt128(MI, MBB, false, SystemZ::subreg_low); 2497 case SystemZ::ZEXT128_32: 2498 return emitExt128(MI, MBB, true, SystemZ::subreg_low32); 2499 case SystemZ::ZEXT128_64: 2500 return emitExt128(MI, MBB, true, SystemZ::subreg_low); 2501 2502 case SystemZ::ATOMIC_SWAPW: 2503 return emitAtomicLoadBinary(MI, MBB, 0, 0); 2504 case SystemZ::ATOMIC_SWAP_32: 2505 return emitAtomicLoadBinary(MI, MBB, 0, 32); 2506 case SystemZ::ATOMIC_SWAP_64: 2507 return emitAtomicLoadBinary(MI, MBB, 0, 64); 2508 2509 case SystemZ::ATOMIC_LOADW_AR: 2510 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 2511 case SystemZ::ATOMIC_LOADW_AFI: 2512 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 2513 case SystemZ::ATOMIC_LOAD_AR: 2514 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 2515 case SystemZ::ATOMIC_LOAD_AHI: 2516 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 2517 case SystemZ::ATOMIC_LOAD_AFI: 2518 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 2519 case SystemZ::ATOMIC_LOAD_AGR: 2520 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 2521 case SystemZ::ATOMIC_LOAD_AGHI: 2522 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 2523 case SystemZ::ATOMIC_LOAD_AGFI: 2524 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 2525 2526 case SystemZ::ATOMIC_LOADW_SR: 2527 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 2528 case SystemZ::ATOMIC_LOAD_SR: 2529 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 2530 case SystemZ::ATOMIC_LOAD_SGR: 2531 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 2532 2533 case SystemZ::ATOMIC_LOADW_NR: 2534 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 2535 case SystemZ::ATOMIC_LOADW_NILH: 2536 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0); 2537 case SystemZ::ATOMIC_LOAD_NR: 2538 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 2539 case SystemZ::ATOMIC_LOAD_NILL32: 2540 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32); 2541 case SystemZ::ATOMIC_LOAD_NILH32: 2542 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32); 2543 case SystemZ::ATOMIC_LOAD_NILF32: 2544 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32); 2545 case SystemZ::ATOMIC_LOAD_NGR: 2546 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 2547 case SystemZ::ATOMIC_LOAD_NILL: 2548 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64); 2549 case SystemZ::ATOMIC_LOAD_NILH: 2550 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64); 2551 case SystemZ::ATOMIC_LOAD_NIHL: 2552 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); 2553 case SystemZ::ATOMIC_LOAD_NIHH: 2554 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); 2555 case SystemZ::ATOMIC_LOAD_NILF: 2556 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64); 2557 case SystemZ::ATOMIC_LOAD_NIHF: 2558 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); 2559 2560 case SystemZ::ATOMIC_LOADW_OR: 2561 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 2562 case SystemZ::ATOMIC_LOADW_OILH: 2563 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 0); 2564 case SystemZ::ATOMIC_LOAD_OR: 2565 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 2566 case SystemZ::ATOMIC_LOAD_OILL32: 2567 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL32, 32); 2568 case SystemZ::ATOMIC_LOAD_OILH32: 2569 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 32); 2570 case SystemZ::ATOMIC_LOAD_OILF32: 2571 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF32, 32); 2572 case SystemZ::ATOMIC_LOAD_OGR: 2573 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 2574 case SystemZ::ATOMIC_LOAD_OILL: 2575 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 64); 2576 case SystemZ::ATOMIC_LOAD_OILH: 2577 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 64); 2578 case SystemZ::ATOMIC_LOAD_OIHL: 2579 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL, 64); 2580 case SystemZ::ATOMIC_LOAD_OIHH: 2581 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH, 64); 2582 case SystemZ::ATOMIC_LOAD_OILF: 2583 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 64); 2584 case SystemZ::ATOMIC_LOAD_OIHF: 2585 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF, 64); 2586 2587 case SystemZ::ATOMIC_LOADW_XR: 2588 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 2589 case SystemZ::ATOMIC_LOADW_XILF: 2590 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 0); 2591 case SystemZ::ATOMIC_LOAD_XR: 2592 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 2593 case SystemZ::ATOMIC_LOAD_XILF32: 2594 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 32); 2595 case SystemZ::ATOMIC_LOAD_XGR: 2596 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 2597 case SystemZ::ATOMIC_LOAD_XILF: 2598 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 64); 2599 case SystemZ::ATOMIC_LOAD_XIHF: 2600 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); 2601 2602 case SystemZ::ATOMIC_LOADW_NRi: 2603 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 2604 case SystemZ::ATOMIC_LOADW_NILHi: 2605 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0, true); 2606 case SystemZ::ATOMIC_LOAD_NRi: 2607 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 2608 case SystemZ::ATOMIC_LOAD_NILL32i: 2609 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32, true); 2610 case SystemZ::ATOMIC_LOAD_NILH32i: 2611 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32, true); 2612 case SystemZ::ATOMIC_LOAD_NILF32i: 2613 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32, true); 2614 case SystemZ::ATOMIC_LOAD_NGRi: 2615 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 2616 case SystemZ::ATOMIC_LOAD_NILLi: 2617 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64, true); 2618 case SystemZ::ATOMIC_LOAD_NILHi: 2619 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64, true); 2620 case SystemZ::ATOMIC_LOAD_NIHLi: 2621 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); 2622 case SystemZ::ATOMIC_LOAD_NIHHi: 2623 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); 2624 case SystemZ::ATOMIC_LOAD_NILFi: 2625 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64, true); 2626 case SystemZ::ATOMIC_LOAD_NIHFi: 2627 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); 2628 2629 case SystemZ::ATOMIC_LOADW_MIN: 2630 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2631 SystemZ::CCMASK_CMP_LE, 0); 2632 case SystemZ::ATOMIC_LOAD_MIN_32: 2633 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2634 SystemZ::CCMASK_CMP_LE, 32); 2635 case SystemZ::ATOMIC_LOAD_MIN_64: 2636 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2637 SystemZ::CCMASK_CMP_LE, 64); 2638 2639 case SystemZ::ATOMIC_LOADW_MAX: 2640 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2641 SystemZ::CCMASK_CMP_GE, 0); 2642 case SystemZ::ATOMIC_LOAD_MAX_32: 2643 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2644 SystemZ::CCMASK_CMP_GE, 32); 2645 case SystemZ::ATOMIC_LOAD_MAX_64: 2646 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2647 SystemZ::CCMASK_CMP_GE, 64); 2648 2649 case SystemZ::ATOMIC_LOADW_UMIN: 2650 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2651 SystemZ::CCMASK_CMP_LE, 0); 2652 case SystemZ::ATOMIC_LOAD_UMIN_32: 2653 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2654 SystemZ::CCMASK_CMP_LE, 32); 2655 case SystemZ::ATOMIC_LOAD_UMIN_64: 2656 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2657 SystemZ::CCMASK_CMP_LE, 64); 2658 2659 case SystemZ::ATOMIC_LOADW_UMAX: 2660 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2661 SystemZ::CCMASK_CMP_GE, 0); 2662 case SystemZ::ATOMIC_LOAD_UMAX_32: 2663 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2664 SystemZ::CCMASK_CMP_GE, 32); 2665 case SystemZ::ATOMIC_LOAD_UMAX_64: 2666 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2667 SystemZ::CCMASK_CMP_GE, 64); 2668 2669 case SystemZ::ATOMIC_CMP_SWAPW: 2670 return emitAtomicCmpSwapW(MI, MBB); 2671 case SystemZ::MVCWrapper: 2672 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 2673 case SystemZ::CLCWrapper: 2674 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 2675 case SystemZ::CLSTLoop: 2676 return emitStringWrapper(MI, MBB, SystemZ::CLST); 2677 case SystemZ::MVSTLoop: 2678 return emitStringWrapper(MI, MBB, SystemZ::MVST); 2679 case SystemZ::SRSTLoop: 2680 return emitStringWrapper(MI, MBB, SystemZ::SRST); 2681 default: 2682 llvm_unreachable("Unexpected instr type to insert"); 2683 } 2684} 2685