SystemZISelLowering.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "SystemZISelLowering.h" 15#include "SystemZCallingConv.h" 16#include "SystemZConstantPoolValue.h" 17#include "SystemZMachineFunctionInfo.h" 18#include "SystemZTargetMachine.h" 19#include "llvm/CodeGen/CallingConvLower.h" 20#include "llvm/CodeGen/MachineInstrBuilder.h" 21#include "llvm/CodeGen/MachineRegisterInfo.h" 22#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 23#include <cctype> 24 25using namespace llvm; 26 27#define DEBUG_TYPE "systemz-lower" 28 29namespace { 30// Represents a sequence for extracting a 0/1 value from an IPM result: 31// (((X ^ XORValue) + AddValue) >> Bit) 32struct IPMConversion { 33 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 34 : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 35 36 int64_t XORValue; 37 int64_t AddValue; 38 unsigned Bit; 39}; 40 41// Represents information about a comparison. 42struct Comparison { 43 Comparison(SDValue Op0In, SDValue Op1In) 44 : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {} 45 46 // The operands to the comparison. 47 SDValue Op0, Op1; 48 49 // The opcode that should be used to compare Op0 and Op1. 50 unsigned Opcode; 51 52 // A SystemZICMP value. Only used for integer comparisons. 53 unsigned ICmpType; 54 55 // The mask of CC values that Opcode can produce. 56 unsigned CCValid; 57 58 // The mask of CC values for which the original condition is true. 59 unsigned CCMask; 60}; 61} // end anonymous namespace 62 63// Classify VT as either 32 or 64 bit. 64static bool is32Bit(EVT VT) { 65 switch (VT.getSimpleVT().SimpleTy) { 66 case MVT::i32: 67 return true; 68 case MVT::i64: 69 return false; 70 default: 71 llvm_unreachable("Unsupported type"); 72 } 73} 74 75// Return a version of MachineOperand that can be safely used before the 76// final use. 77static MachineOperand earlyUseOperand(MachineOperand Op) { 78 if (Op.isReg()) 79 Op.setIsKill(false); 80 return Op; 81} 82 83SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 84 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 85 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 86 MVT PtrVT = getPointerTy(); 87 88 // Set up the register classes. 89 if (Subtarget.hasHighWord()) 90 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 91 else 92 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 93 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 94 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 95 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 96 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 97 98 // Compute derived properties from the register classes 99 computeRegisterProperties(); 100 101 // Set up special registers. 102 setExceptionPointerRegister(SystemZ::R6D); 103 setExceptionSelectorRegister(SystemZ::R7D); 104 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 105 106 // TODO: It may be better to default to latency-oriented scheduling, however 107 // LLVM's current latency-oriented scheduler can't handle physreg definitions 108 // such as SystemZ has with CC, so set this to the register-pressure 109 // scheduler, because it can. 110 setSchedulingPreference(Sched::RegPressure); 111 112 setBooleanContents(ZeroOrOneBooleanContent); 113 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 114 115 // Instructions are strings of 2-byte aligned 2-byte values. 116 setMinFunctionAlignment(2); 117 118 // Handle operations that are handled in a similar way for all types. 119 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 120 I <= MVT::LAST_FP_VALUETYPE; 121 ++I) { 122 MVT VT = MVT::SimpleValueType(I); 123 if (isTypeLegal(VT)) { 124 // Lower SET_CC into an IPM-based sequence. 125 setOperationAction(ISD::SETCC, VT, Custom); 126 127 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 128 setOperationAction(ISD::SELECT, VT, Expand); 129 130 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 131 setOperationAction(ISD::SELECT_CC, VT, Custom); 132 setOperationAction(ISD::BR_CC, VT, Custom); 133 } 134 } 135 136 // Expand jump table branches as address arithmetic followed by an 137 // indirect jump. 138 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 139 140 // Expand BRCOND into a BR_CC (see above). 141 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 142 143 // Handle integer types. 144 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 145 I <= MVT::LAST_INTEGER_VALUETYPE; 146 ++I) { 147 MVT VT = MVT::SimpleValueType(I); 148 if (isTypeLegal(VT)) { 149 // Expand individual DIV and REMs into DIVREMs. 150 setOperationAction(ISD::SDIV, VT, Expand); 151 setOperationAction(ISD::UDIV, VT, Expand); 152 setOperationAction(ISD::SREM, VT, Expand); 153 setOperationAction(ISD::UREM, VT, Expand); 154 setOperationAction(ISD::SDIVREM, VT, Custom); 155 setOperationAction(ISD::UDIVREM, VT, Custom); 156 157 // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and 158 // stores, putting a serialization instruction after the stores. 159 setOperationAction(ISD::ATOMIC_LOAD, VT, Custom); 160 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 161 162 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are 163 // available, or if the operand is constant. 164 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 165 166 // No special instructions for these. 167 setOperationAction(ISD::CTPOP, VT, Expand); 168 setOperationAction(ISD::CTTZ, VT, Expand); 169 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 170 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 171 setOperationAction(ISD::ROTR, VT, Expand); 172 173 // Use *MUL_LOHI where possible instead of MULH*. 174 setOperationAction(ISD::MULHS, VT, Expand); 175 setOperationAction(ISD::MULHU, VT, Expand); 176 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 177 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 178 179 // Only z196 and above have native support for conversions to unsigned. 180 if (!Subtarget.hasFPExtension()) 181 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 182 } 183 } 184 185 // Type legalization will convert 8- and 16-bit atomic operations into 186 // forms that operate on i32s (but still keeping the original memory VT). 187 // Lower them into full i32 operations. 188 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 189 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 190 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 191 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 192 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 193 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 194 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 195 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 196 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 197 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 198 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 199 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 200 201 // z10 has instructions for signed but not unsigned FP conversion. 202 // Handle unsigned 32-bit types as signed 64-bit types. 203 if (!Subtarget.hasFPExtension()) { 204 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 205 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 206 } 207 208 // We have native support for a 64-bit CTLZ, via FLOGR. 209 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 210 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 211 212 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 213 setOperationAction(ISD::OR, MVT::i64, Custom); 214 215 // FIXME: Can we support these natively? 216 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 217 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 218 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 219 220 // We have native instructions for i8, i16 and i32 extensions, but not i1. 221 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 222 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 223 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 224 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 225 226 // Handle the various types of symbolic address. 227 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 228 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 229 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 230 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 231 setOperationAction(ISD::JumpTable, PtrVT, Custom); 232 233 // We need to handle dynamic allocations specially because of the 234 // 160-byte area at the bottom of the stack. 235 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 236 237 // Use custom expanders so that we can force the function to use 238 // a frame pointer. 239 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 240 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 241 242 // Handle prefetches with PFD or PFDRL. 243 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 244 245 // Handle floating-point types. 246 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 247 I <= MVT::LAST_FP_VALUETYPE; 248 ++I) { 249 MVT VT = MVT::SimpleValueType(I); 250 if (isTypeLegal(VT)) { 251 // We can use FI for FRINT. 252 setOperationAction(ISD::FRINT, VT, Legal); 253 254 // We can use the extended form of FI for other rounding operations. 255 if (Subtarget.hasFPExtension()) { 256 setOperationAction(ISD::FNEARBYINT, VT, Legal); 257 setOperationAction(ISD::FFLOOR, VT, Legal); 258 setOperationAction(ISD::FCEIL, VT, Legal); 259 setOperationAction(ISD::FTRUNC, VT, Legal); 260 setOperationAction(ISD::FROUND, VT, Legal); 261 } 262 263 // No special instructions for these. 264 setOperationAction(ISD::FSIN, VT, Expand); 265 setOperationAction(ISD::FCOS, VT, Expand); 266 setOperationAction(ISD::FREM, VT, Expand); 267 } 268 } 269 270 // We have fused multiply-addition for f32 and f64 but not f128. 271 setOperationAction(ISD::FMA, MVT::f32, Legal); 272 setOperationAction(ISD::FMA, MVT::f64, Legal); 273 setOperationAction(ISD::FMA, MVT::f128, Expand); 274 275 // Needed so that we don't try to implement f128 constant loads using 276 // a load-and-extend of a f80 constant (in cases where the constant 277 // would fit in an f80). 278 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 279 280 // Floating-point truncation and stores need to be done separately. 281 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 282 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 283 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 284 285 // We have 64-bit FPR<->GPR moves, but need special handling for 286 // 32-bit forms. 287 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 288 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 289 290 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 291 // structure, but VAEND is a no-op. 292 setOperationAction(ISD::VASTART, MVT::Other, Custom); 293 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 294 setOperationAction(ISD::VAEND, MVT::Other, Expand); 295 296 // Codes for which we want to perform some z-specific combinations. 297 setTargetDAGCombine(ISD::SIGN_EXTEND); 298 299 // We want to use MVC in preference to even a single load/store pair. 300 MaxStoresPerMemcpy = 0; 301 MaxStoresPerMemcpyOptSize = 0; 302 303 // The main memset sequence is a byte store followed by an MVC. 304 // Two STC or MV..I stores win over that, but the kind of fused stores 305 // generated by target-independent code don't when the byte value is 306 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 307 // than "STC;MVC". Handle the choice in target-specific code instead. 308 MaxStoresPerMemset = 0; 309 MaxStoresPerMemsetOptSize = 0; 310} 311 312EVT SystemZTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { 313 if (!VT.isVector()) 314 return MVT::i32; 315 return VT.changeVectorElementTypeToInteger(); 316} 317 318bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 319 VT = VT.getScalarType(); 320 321 if (!VT.isSimple()) 322 return false; 323 324 switch (VT.getSimpleVT().SimpleTy) { 325 case MVT::f32: 326 case MVT::f64: 327 return true; 328 case MVT::f128: 329 return false; 330 default: 331 break; 332 } 333 334 return false; 335} 336 337bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 338 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 339 return Imm.isZero() || Imm.isNegZero(); 340} 341 342bool SystemZTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 343 unsigned, 344 bool *Fast) const { 345 // Unaligned accesses should never be slower than the expanded version. 346 // We check specifically for aligned accesses in the few cases where 347 // they are required. 348 if (Fast) 349 *Fast = true; 350 return true; 351} 352 353bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 354 Type *Ty) const { 355 // Punt on globals for now, although they can be used in limited 356 // RELATIVE LONG cases. 357 if (AM.BaseGV) 358 return false; 359 360 // Require a 20-bit signed offset. 361 if (!isInt<20>(AM.BaseOffs)) 362 return false; 363 364 // Indexing is OK but no scale factor can be applied. 365 return AM.Scale == 0 || AM.Scale == 1; 366} 367 368bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 369 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 370 return false; 371 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 372 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 373 return FromBits > ToBits; 374} 375 376bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 377 if (!FromVT.isInteger() || !ToVT.isInteger()) 378 return false; 379 unsigned FromBits = FromVT.getSizeInBits(); 380 unsigned ToBits = ToVT.getSizeInBits(); 381 return FromBits > ToBits; 382} 383 384//===----------------------------------------------------------------------===// 385// Inline asm support 386//===----------------------------------------------------------------------===// 387 388TargetLowering::ConstraintType 389SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 390 if (Constraint.size() == 1) { 391 switch (Constraint[0]) { 392 case 'a': // Address register 393 case 'd': // Data register (equivalent to 'r') 394 case 'f': // Floating-point register 395 case 'h': // High-part register 396 case 'r': // General-purpose register 397 return C_RegisterClass; 398 399 case 'Q': // Memory with base and unsigned 12-bit displacement 400 case 'R': // Likewise, plus an index 401 case 'S': // Memory with base and signed 20-bit displacement 402 case 'T': // Likewise, plus an index 403 case 'm': // Equivalent to 'T'. 404 return C_Memory; 405 406 case 'I': // Unsigned 8-bit constant 407 case 'J': // Unsigned 12-bit constant 408 case 'K': // Signed 16-bit constant 409 case 'L': // Signed 20-bit displacement (on all targets we support) 410 case 'M': // 0x7fffffff 411 return C_Other; 412 413 default: 414 break; 415 } 416 } 417 return TargetLowering::getConstraintType(Constraint); 418} 419 420TargetLowering::ConstraintWeight SystemZTargetLowering:: 421getSingleConstraintMatchWeight(AsmOperandInfo &info, 422 const char *constraint) const { 423 ConstraintWeight weight = CW_Invalid; 424 Value *CallOperandVal = info.CallOperandVal; 425 // If we don't have a value, we can't do a match, 426 // but allow it at the lowest weight. 427 if (!CallOperandVal) 428 return CW_Default; 429 Type *type = CallOperandVal->getType(); 430 // Look at the constraint type. 431 switch (*constraint) { 432 default: 433 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 434 break; 435 436 case 'a': // Address register 437 case 'd': // Data register (equivalent to 'r') 438 case 'h': // High-part register 439 case 'r': // General-purpose register 440 if (CallOperandVal->getType()->isIntegerTy()) 441 weight = CW_Register; 442 break; 443 444 case 'f': // Floating-point register 445 if (type->isFloatingPointTy()) 446 weight = CW_Register; 447 break; 448 449 case 'I': // Unsigned 8-bit constant 450 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 451 if (isUInt<8>(C->getZExtValue())) 452 weight = CW_Constant; 453 break; 454 455 case 'J': // Unsigned 12-bit constant 456 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 457 if (isUInt<12>(C->getZExtValue())) 458 weight = CW_Constant; 459 break; 460 461 case 'K': // Signed 16-bit constant 462 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 463 if (isInt<16>(C->getSExtValue())) 464 weight = CW_Constant; 465 break; 466 467 case 'L': // Signed 20-bit displacement (on all targets we support) 468 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 469 if (isInt<20>(C->getSExtValue())) 470 weight = CW_Constant; 471 break; 472 473 case 'M': // 0x7fffffff 474 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 475 if (C->getZExtValue() == 0x7fffffff) 476 weight = CW_Constant; 477 break; 478 } 479 return weight; 480} 481 482// Parse a "{tNNN}" register constraint for which the register type "t" 483// has already been verified. MC is the class associated with "t" and 484// Map maps 0-based register numbers to LLVM register numbers. 485static std::pair<unsigned, const TargetRegisterClass *> 486parseRegisterNumber(const std::string &Constraint, 487 const TargetRegisterClass *RC, const unsigned *Map) { 488 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 489 if (isdigit(Constraint[2])) { 490 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 491 unsigned Index = atoi(Suffix.c_str()); 492 if (Index < 16 && Map[Index]) 493 return std::make_pair(Map[Index], RC); 494 } 495 return std::make_pair(0U, nullptr); 496} 497 498std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 499getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 500 if (Constraint.size() == 1) { 501 // GCC Constraint Letters 502 switch (Constraint[0]) { 503 default: break; 504 case 'd': // Data register (equivalent to 'r') 505 case 'r': // General-purpose register 506 if (VT == MVT::i64) 507 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 508 else if (VT == MVT::i128) 509 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 510 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 511 512 case 'a': // Address register 513 if (VT == MVT::i64) 514 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 515 else if (VT == MVT::i128) 516 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 517 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 518 519 case 'h': // High-part register (an LLVM extension) 520 return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 521 522 case 'f': // Floating-point register 523 if (VT == MVT::f64) 524 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 525 else if (VT == MVT::f128) 526 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 527 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 528 } 529 } 530 if (Constraint[0] == '{') { 531 // We need to override the default register parsing for GPRs and FPRs 532 // because the interpretation depends on VT. The internal names of 533 // the registers are also different from the external names 534 // (F0D and F0S instead of F0, etc.). 535 if (Constraint[1] == 'r') { 536 if (VT == MVT::i32) 537 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 538 SystemZMC::GR32Regs); 539 if (VT == MVT::i128) 540 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 541 SystemZMC::GR128Regs); 542 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 543 SystemZMC::GR64Regs); 544 } 545 if (Constraint[1] == 'f') { 546 if (VT == MVT::f32) 547 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 548 SystemZMC::FP32Regs); 549 if (VT == MVT::f128) 550 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 551 SystemZMC::FP128Regs); 552 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 553 SystemZMC::FP64Regs); 554 } 555 } 556 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 557} 558 559void SystemZTargetLowering:: 560LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 561 std::vector<SDValue> &Ops, 562 SelectionDAG &DAG) const { 563 // Only support length 1 constraints for now. 564 if (Constraint.length() == 1) { 565 switch (Constraint[0]) { 566 case 'I': // Unsigned 8-bit constant 567 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 568 if (isUInt<8>(C->getZExtValue())) 569 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 570 Op.getValueType())); 571 return; 572 573 case 'J': // Unsigned 12-bit constant 574 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 575 if (isUInt<12>(C->getZExtValue())) 576 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 577 Op.getValueType())); 578 return; 579 580 case 'K': // Signed 16-bit constant 581 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 582 if (isInt<16>(C->getSExtValue())) 583 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 584 Op.getValueType())); 585 return; 586 587 case 'L': // Signed 20-bit displacement (on all targets we support) 588 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 589 if (isInt<20>(C->getSExtValue())) 590 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 591 Op.getValueType())); 592 return; 593 594 case 'M': // 0x7fffffff 595 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 596 if (C->getZExtValue() == 0x7fffffff) 597 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 598 Op.getValueType())); 599 return; 600 } 601 } 602 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 603} 604 605//===----------------------------------------------------------------------===// 606// Calling conventions 607//===----------------------------------------------------------------------===// 608 609#include "SystemZGenCallingConv.inc" 610 611bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 612 Type *ToType) const { 613 return isTruncateFree(FromType, ToType); 614} 615 616bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 617 if (!CI->isTailCall()) 618 return false; 619 return true; 620} 621 622// Value is a value that has been passed to us in the location described by VA 623// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 624// any loads onto Chain. 625static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 626 CCValAssign &VA, SDValue Chain, 627 SDValue Value) { 628 // If the argument has been promoted from a smaller type, insert an 629 // assertion to capture this. 630 if (VA.getLocInfo() == CCValAssign::SExt) 631 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 632 DAG.getValueType(VA.getValVT())); 633 else if (VA.getLocInfo() == CCValAssign::ZExt) 634 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 635 DAG.getValueType(VA.getValVT())); 636 637 if (VA.isExtInLoc()) 638 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 639 else if (VA.getLocInfo() == CCValAssign::Indirect) 640 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 641 MachinePointerInfo(), false, false, false, 0); 642 else 643 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 644 return Value; 645} 646 647// Value is a value of type VA.getValVT() that we need to copy into 648// the location described by VA. Return a copy of Value converted to 649// VA.getValVT(). The caller is responsible for handling indirect values. 650static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 651 CCValAssign &VA, SDValue Value) { 652 switch (VA.getLocInfo()) { 653 case CCValAssign::SExt: 654 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 655 case CCValAssign::ZExt: 656 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 657 case CCValAssign::AExt: 658 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 659 case CCValAssign::Full: 660 return Value; 661 default: 662 llvm_unreachable("Unhandled getLocInfo()"); 663 } 664} 665 666SDValue SystemZTargetLowering:: 667LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 668 const SmallVectorImpl<ISD::InputArg> &Ins, 669 SDLoc DL, SelectionDAG &DAG, 670 SmallVectorImpl<SDValue> &InVals) const { 671 MachineFunction &MF = DAG.getMachineFunction(); 672 MachineFrameInfo *MFI = MF.getFrameInfo(); 673 MachineRegisterInfo &MRI = MF.getRegInfo(); 674 SystemZMachineFunctionInfo *FuncInfo = 675 MF.getInfo<SystemZMachineFunctionInfo>(); 676 auto *TFL = static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 677 678 // Assign locations to all of the incoming arguments. 679 SmallVector<CCValAssign, 16> ArgLocs; 680 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 681 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 682 683 unsigned NumFixedGPRs = 0; 684 unsigned NumFixedFPRs = 0; 685 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 686 SDValue ArgValue; 687 CCValAssign &VA = ArgLocs[I]; 688 EVT LocVT = VA.getLocVT(); 689 if (VA.isRegLoc()) { 690 // Arguments passed in registers 691 const TargetRegisterClass *RC; 692 switch (LocVT.getSimpleVT().SimpleTy) { 693 default: 694 // Integers smaller than i64 should be promoted to i64. 695 llvm_unreachable("Unexpected argument type"); 696 case MVT::i32: 697 NumFixedGPRs += 1; 698 RC = &SystemZ::GR32BitRegClass; 699 break; 700 case MVT::i64: 701 NumFixedGPRs += 1; 702 RC = &SystemZ::GR64BitRegClass; 703 break; 704 case MVT::f32: 705 NumFixedFPRs += 1; 706 RC = &SystemZ::FP32BitRegClass; 707 break; 708 case MVT::f64: 709 NumFixedFPRs += 1; 710 RC = &SystemZ::FP64BitRegClass; 711 break; 712 } 713 714 unsigned VReg = MRI.createVirtualRegister(RC); 715 MRI.addLiveIn(VA.getLocReg(), VReg); 716 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 717 } else { 718 assert(VA.isMemLoc() && "Argument not register or memory"); 719 720 // Create the frame index object for this incoming parameter. 721 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 722 VA.getLocMemOffset(), true); 723 724 // Create the SelectionDAG nodes corresponding to a load 725 // from this parameter. Unpromoted ints and floats are 726 // passed as right-justified 8-byte values. 727 EVT PtrVT = getPointerTy(); 728 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 729 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 730 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 731 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 732 MachinePointerInfo::getFixedStack(FI), 733 false, false, false, 0); 734 } 735 736 // Convert the value of the argument register into the value that's 737 // being passed. 738 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 739 } 740 741 if (IsVarArg) { 742 // Save the number of non-varargs registers for later use by va_start, etc. 743 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 744 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 745 746 // Likewise the address (in the form of a frame index) of where the 747 // first stack vararg would be. The 1-byte size here is arbitrary. 748 int64_t StackSize = CCInfo.getNextStackOffset(); 749 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 750 751 // ...and a similar frame index for the caller-allocated save area 752 // that will be used to store the incoming registers. 753 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 754 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 755 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 756 757 // Store the FPR varargs in the reserved frame slots. (We store the 758 // GPRs as part of the prologue.) 759 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 760 SDValue MemOps[SystemZ::NumArgFPRs]; 761 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 762 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 763 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 764 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 765 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 766 &SystemZ::FP64BitRegClass); 767 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 768 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 769 MachinePointerInfo::getFixedStack(FI), 770 false, false, 0); 771 772 } 773 // Join the stores, which are independent of one another. 774 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 775 makeArrayRef(&MemOps[NumFixedFPRs], 776 SystemZ::NumArgFPRs-NumFixedFPRs)); 777 } 778 } 779 780 return Chain; 781} 782 783static bool canUseSiblingCall(CCState ArgCCInfo, 784 SmallVectorImpl<CCValAssign> &ArgLocs) { 785 // Punt if there are any indirect or stack arguments, or if the call 786 // needs the call-saved argument register R6. 787 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 788 CCValAssign &VA = ArgLocs[I]; 789 if (VA.getLocInfo() == CCValAssign::Indirect) 790 return false; 791 if (!VA.isRegLoc()) 792 return false; 793 unsigned Reg = VA.getLocReg(); 794 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 795 return false; 796 } 797 return true; 798} 799 800SDValue 801SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 802 SmallVectorImpl<SDValue> &InVals) const { 803 SelectionDAG &DAG = CLI.DAG; 804 SDLoc &DL = CLI.DL; 805 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 806 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 807 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 808 SDValue Chain = CLI.Chain; 809 SDValue Callee = CLI.Callee; 810 bool &IsTailCall = CLI.IsTailCall; 811 CallingConv::ID CallConv = CLI.CallConv; 812 bool IsVarArg = CLI.IsVarArg; 813 MachineFunction &MF = DAG.getMachineFunction(); 814 EVT PtrVT = getPointerTy(); 815 816 // Analyze the operands of the call, assigning locations to each operand. 817 SmallVector<CCValAssign, 16> ArgLocs; 818 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 819 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 820 821 // We don't support GuaranteedTailCallOpt, only automatically-detected 822 // sibling calls. 823 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 824 IsTailCall = false; 825 826 // Get a count of how many bytes are to be pushed on the stack. 827 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 828 829 // Mark the start of the call. 830 if (!IsTailCall) 831 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 832 DL); 833 834 // Copy argument values to their designated locations. 835 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 836 SmallVector<SDValue, 8> MemOpChains; 837 SDValue StackPtr; 838 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 839 CCValAssign &VA = ArgLocs[I]; 840 SDValue ArgValue = OutVals[I]; 841 842 if (VA.getLocInfo() == CCValAssign::Indirect) { 843 // Store the argument in a stack slot and pass its address. 844 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 845 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 846 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 847 MachinePointerInfo::getFixedStack(FI), 848 false, false, 0)); 849 ArgValue = SpillSlot; 850 } else 851 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 852 853 if (VA.isRegLoc()) 854 // Queue up the argument copies and emit them at the end. 855 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 856 else { 857 assert(VA.isMemLoc() && "Argument not register or memory"); 858 859 // Work out the address of the stack slot. Unpromoted ints and 860 // floats are passed as right-justified 8-byte values. 861 if (!StackPtr.getNode()) 862 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 863 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 864 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 865 Offset += 4; 866 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 867 DAG.getIntPtrConstant(Offset)); 868 869 // Emit the store. 870 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 871 MachinePointerInfo(), 872 false, false, 0)); 873 } 874 } 875 876 // Join the stores, which are independent of one another. 877 if (!MemOpChains.empty()) 878 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 879 880 // Accept direct calls by converting symbolic call addresses to the 881 // associated Target* opcodes. Force %r1 to be used for indirect 882 // tail calls. 883 SDValue Glue; 884 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 885 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 886 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 887 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 888 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 889 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 890 } else if (IsTailCall) { 891 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 892 Glue = Chain.getValue(1); 893 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 894 } 895 896 // Build a sequence of copy-to-reg nodes, chained and glued together. 897 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 898 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 899 RegsToPass[I].second, Glue); 900 Glue = Chain.getValue(1); 901 } 902 903 // The first call operand is the chain and the second is the target address. 904 SmallVector<SDValue, 8> Ops; 905 Ops.push_back(Chain); 906 Ops.push_back(Callee); 907 908 // Add argument registers to the end of the list so that they are 909 // known live into the call. 910 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 911 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 912 RegsToPass[I].second.getValueType())); 913 914 // Glue the call to the argument copies, if any. 915 if (Glue.getNode()) 916 Ops.push_back(Glue); 917 918 // Emit the call. 919 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 920 if (IsTailCall) 921 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops); 922 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops); 923 Glue = Chain.getValue(1); 924 925 // Mark the end of the call, which is glued to the call itself. 926 Chain = DAG.getCALLSEQ_END(Chain, 927 DAG.getConstant(NumBytes, PtrVT, true), 928 DAG.getConstant(0, PtrVT, true), 929 Glue, DL); 930 Glue = Chain.getValue(1); 931 932 // Assign locations to each value returned by this call. 933 SmallVector<CCValAssign, 16> RetLocs; 934 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 935 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 936 937 // Copy all of the result registers out of their specified physreg. 938 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 939 CCValAssign &VA = RetLocs[I]; 940 941 // Copy the value out, gluing the copy to the end of the call sequence. 942 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 943 VA.getLocVT(), Glue); 944 Chain = RetValue.getValue(1); 945 Glue = RetValue.getValue(2); 946 947 // Convert the value of the return register into the value that's 948 // being returned. 949 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 950 } 951 952 return Chain; 953} 954 955SDValue 956SystemZTargetLowering::LowerReturn(SDValue Chain, 957 CallingConv::ID CallConv, bool IsVarArg, 958 const SmallVectorImpl<ISD::OutputArg> &Outs, 959 const SmallVectorImpl<SDValue> &OutVals, 960 SDLoc DL, SelectionDAG &DAG) const { 961 MachineFunction &MF = DAG.getMachineFunction(); 962 963 // Assign locations to each returned value. 964 SmallVector<CCValAssign, 16> RetLocs; 965 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 966 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 967 968 // Quick exit for void returns 969 if (RetLocs.empty()) 970 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 971 972 // Copy the result values into the output registers. 973 SDValue Glue; 974 SmallVector<SDValue, 4> RetOps; 975 RetOps.push_back(Chain); 976 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 977 CCValAssign &VA = RetLocs[I]; 978 SDValue RetValue = OutVals[I]; 979 980 // Make the return register live on exit. 981 assert(VA.isRegLoc() && "Can only return in registers!"); 982 983 // Promote the value as required. 984 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 985 986 // Chain and glue the copies together. 987 unsigned Reg = VA.getLocReg(); 988 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 989 Glue = Chain.getValue(1); 990 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 991 } 992 993 // Update chain and glue. 994 RetOps[0] = Chain; 995 if (Glue.getNode()) 996 RetOps.push_back(Glue); 997 998 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps); 999} 1000 1001SDValue SystemZTargetLowering:: 1002prepareVolatileOrAtomicLoad(SDValue Chain, SDLoc DL, SelectionDAG &DAG) const { 1003 return DAG.getNode(SystemZISD::SERIALIZE, DL, MVT::Other, Chain); 1004} 1005 1006// CC is a comparison that will be implemented using an integer or 1007// floating-point comparison. Return the condition code mask for 1008// a branch on true. In the integer case, CCMASK_CMP_UO is set for 1009// unsigned comparisons and clear for signed ones. In the floating-point 1010// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 1011static unsigned CCMaskForCondCode(ISD::CondCode CC) { 1012#define CONV(X) \ 1013 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 1014 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 1015 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 1016 1017 switch (CC) { 1018 default: 1019 llvm_unreachable("Invalid integer condition!"); 1020 1021 CONV(EQ); 1022 CONV(NE); 1023 CONV(GT); 1024 CONV(GE); 1025 CONV(LT); 1026 CONV(LE); 1027 1028 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 1029 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 1030 } 1031#undef CONV 1032} 1033 1034// Return a sequence for getting a 1 from an IPM result when CC has a 1035// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1036// The handling of CC values outside CCValid doesn't matter. 1037static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1038 // Deal with cases where the result can be taken directly from a bit 1039 // of the IPM result. 1040 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1041 return IPMConversion(0, 0, SystemZ::IPM_CC); 1042 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1043 return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1044 1045 // Deal with cases where we can add a value to force the sign bit 1046 // to contain the right value. Putting the bit in 31 means we can 1047 // use SRL rather than RISBG(L), and also makes it easier to get a 1048 // 0/-1 value, so it has priority over the other tests below. 1049 // 1050 // These sequences rely on the fact that the upper two bits of the 1051 // IPM result are zero. 1052 uint64_t TopBit = uint64_t(1) << 31; 1053 if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1054 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1055 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1056 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1057 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1058 | SystemZ::CCMASK_1 1059 | SystemZ::CCMASK_2))) 1060 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1061 if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1062 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1063 if (CCMask == (CCValid & (SystemZ::CCMASK_1 1064 | SystemZ::CCMASK_2 1065 | SystemZ::CCMASK_3))) 1066 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1067 1068 // Next try inverting the value and testing a bit. 0/1 could be 1069 // handled this way too, but we dealt with that case above. 1070 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1071 return IPMConversion(-1, 0, SystemZ::IPM_CC); 1072 1073 // Handle cases where adding a value forces a non-sign bit to contain 1074 // the right value. 1075 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1076 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1077 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1078 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1079 1080 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are 1081 // can be done by inverting the low CC bit and applying one of the 1082 // sign-based extractions above. 1083 if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1084 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1085 if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1086 return IPMConversion(1 << SystemZ::IPM_CC, 1087 TopBit - (3 << SystemZ::IPM_CC), 31); 1088 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1089 | SystemZ::CCMASK_1 1090 | SystemZ::CCMASK_3))) 1091 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1092 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1093 | SystemZ::CCMASK_2 1094 | SystemZ::CCMASK_3))) 1095 return IPMConversion(1 << SystemZ::IPM_CC, 1096 TopBit - (1 << SystemZ::IPM_CC), 31); 1097 1098 llvm_unreachable("Unexpected CC combination"); 1099} 1100 1101// If C can be converted to a comparison against zero, adjust the operands 1102// as necessary. 1103static void adjustZeroCmp(SelectionDAG &DAG, Comparison &C) { 1104 if (C.ICmpType == SystemZICMP::UnsignedOnly) 1105 return; 1106 1107 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode()); 1108 if (!ConstOp1) 1109 return; 1110 1111 int64_t Value = ConstOp1->getSExtValue(); 1112 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) || 1113 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) || 1114 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) || 1115 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) { 1116 C.CCMask ^= SystemZ::CCMASK_CMP_EQ; 1117 C.Op1 = DAG.getConstant(0, C.Op1.getValueType()); 1118 } 1119} 1120 1121// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI, 1122// adjust the operands as necessary. 1123static void adjustSubwordCmp(SelectionDAG &DAG, Comparison &C) { 1124 // For us to make any changes, it must a comparison between a single-use 1125 // load and a constant. 1126 if (!C.Op0.hasOneUse() || 1127 C.Op0.getOpcode() != ISD::LOAD || 1128 C.Op1.getOpcode() != ISD::Constant) 1129 return; 1130 1131 // We must have an 8- or 16-bit load. 1132 auto *Load = cast<LoadSDNode>(C.Op0); 1133 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1134 if (NumBits != 8 && NumBits != 16) 1135 return; 1136 1137 // The load must be an extending one and the constant must be within the 1138 // range of the unextended value. 1139 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1); 1140 uint64_t Value = ConstOp1->getZExtValue(); 1141 uint64_t Mask = (1 << NumBits) - 1; 1142 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1143 // Make sure that ConstOp1 is in range of C.Op0. 1144 int64_t SignedValue = ConstOp1->getSExtValue(); 1145 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask) 1146 return; 1147 if (C.ICmpType != SystemZICMP::SignedOnly) { 1148 // Unsigned comparison between two sign-extended values is equivalent 1149 // to unsigned comparison between two zero-extended values. 1150 Value &= Mask; 1151 } else if (NumBits == 8) { 1152 // Try to treat the comparison as unsigned, so that we can use CLI. 1153 // Adjust CCMask and Value as necessary. 1154 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT) 1155 // Test whether the high bit of the byte is set. 1156 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT; 1157 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE) 1158 // Test whether the high bit of the byte is clear. 1159 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT; 1160 else 1161 // No instruction exists for this combination. 1162 return; 1163 C.ICmpType = SystemZICMP::UnsignedOnly; 1164 } 1165 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1166 if (Value > Mask) 1167 return; 1168 assert(C.ICmpType == SystemZICMP::Any && 1169 "Signedness shouldn't matter here."); 1170 } else 1171 return; 1172 1173 // Make sure that the first operand is an i32 of the right extension type. 1174 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ? 1175 ISD::SEXTLOAD : 1176 ISD::ZEXTLOAD); 1177 if (C.Op0.getValueType() != MVT::i32 || 1178 Load->getExtensionType() != ExtType) 1179 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1180 Load->getChain(), Load->getBasePtr(), 1181 Load->getPointerInfo(), Load->getMemoryVT(), 1182 Load->isVolatile(), Load->isNonTemporal(), 1183 Load->getAlignment()); 1184 1185 // Make sure that the second operand is an i32 with the right value. 1186 if (C.Op1.getValueType() != MVT::i32 || 1187 Value != ConstOp1->getZExtValue()) 1188 C.Op1 = DAG.getConstant(Value, MVT::i32); 1189} 1190 1191// Return true if Op is either an unextended load, or a load suitable 1192// for integer register-memory comparisons of type ICmpType. 1193static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 1194 auto *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1195 if (Load) { 1196 // There are no instructions to compare a register with a memory byte. 1197 if (Load->getMemoryVT() == MVT::i8) 1198 return false; 1199 // Otherwise decide on extension type. 1200 switch (Load->getExtensionType()) { 1201 case ISD::NON_EXTLOAD: 1202 return true; 1203 case ISD::SEXTLOAD: 1204 return ICmpType != SystemZICMP::UnsignedOnly; 1205 case ISD::ZEXTLOAD: 1206 return ICmpType != SystemZICMP::SignedOnly; 1207 default: 1208 break; 1209 } 1210 } 1211 return false; 1212} 1213 1214// Return true if it is better to swap the operands of C. 1215static bool shouldSwapCmpOperands(const Comparison &C) { 1216 // Leave f128 comparisons alone, since they have no memory forms. 1217 if (C.Op0.getValueType() == MVT::f128) 1218 return false; 1219 1220 // Always keep a floating-point constant second, since comparisons with 1221 // zero can use LOAD TEST and comparisons with other constants make a 1222 // natural memory operand. 1223 if (isa<ConstantFPSDNode>(C.Op1)) 1224 return false; 1225 1226 // Never swap comparisons with zero since there are many ways to optimize 1227 // those later. 1228 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 1229 if (ConstOp1 && ConstOp1->getZExtValue() == 0) 1230 return false; 1231 1232 // Also keep natural memory operands second if the loaded value is 1233 // only used here. Several comparisons have memory forms. 1234 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse()) 1235 return false; 1236 1237 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1238 // In that case we generally prefer the memory to be second. 1239 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) { 1240 // The only exceptions are when the second operand is a constant and 1241 // we can use things like CHHSI. 1242 if (!ConstOp1) 1243 return true; 1244 // The unsigned memory-immediate instructions can handle 16-bit 1245 // unsigned integers. 1246 if (C.ICmpType != SystemZICMP::SignedOnly && 1247 isUInt<16>(ConstOp1->getZExtValue())) 1248 return false; 1249 // The signed memory-immediate instructions can handle 16-bit 1250 // signed integers. 1251 if (C.ICmpType != SystemZICMP::UnsignedOnly && 1252 isInt<16>(ConstOp1->getSExtValue())) 1253 return false; 1254 return true; 1255 } 1256 1257 // Try to promote the use of CGFR and CLGFR. 1258 unsigned Opcode0 = C.Op0.getOpcode(); 1259 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND) 1260 return true; 1261 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND) 1262 return true; 1263 if (C.ICmpType != SystemZICMP::SignedOnly && 1264 Opcode0 == ISD::AND && 1265 C.Op0.getOperand(1).getOpcode() == ISD::Constant && 1266 cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff) 1267 return true; 1268 1269 return false; 1270} 1271 1272// Return a version of comparison CC mask CCMask in which the LT and GT 1273// actions are swapped. 1274static unsigned reverseCCMask(unsigned CCMask) { 1275 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1276 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1277 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1278 (CCMask & SystemZ::CCMASK_CMP_UO)); 1279} 1280 1281// Check whether C tests for equality between X and Y and whether X - Y 1282// or Y - X is also computed. In that case it's better to compare the 1283// result of the subtraction against zero. 1284static void adjustForSubtraction(SelectionDAG &DAG, Comparison &C) { 1285 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 1286 C.CCMask == SystemZ::CCMASK_CMP_NE) { 1287 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1288 SDNode *N = *I; 1289 if (N->getOpcode() == ISD::SUB && 1290 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) || 1291 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) { 1292 C.Op0 = SDValue(N, 0); 1293 C.Op1 = DAG.getConstant(0, N->getValueType(0)); 1294 return; 1295 } 1296 } 1297 } 1298} 1299 1300// Check whether C compares a floating-point value with zero and if that 1301// floating-point value is also negated. In this case we can use the 1302// negation to set CC, so avoiding separate LOAD AND TEST and 1303// LOAD (NEGATIVE/COMPLEMENT) instructions. 1304static void adjustForFNeg(Comparison &C) { 1305 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1); 1306 if (C1 && C1->isZero()) { 1307 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1308 SDNode *N = *I; 1309 if (N->getOpcode() == ISD::FNEG) { 1310 C.Op0 = SDValue(N, 0); 1311 C.CCMask = reverseCCMask(C.CCMask); 1312 return; 1313 } 1314 } 1315 } 1316} 1317 1318// Check whether C compares (shl X, 32) with 0 and whether X is 1319// also sign-extended. In that case it is better to test the result 1320// of the sign extension using LTGFR. 1321// 1322// This case is important because InstCombine transforms a comparison 1323// with (sext (trunc X)) into a comparison with (shl X, 32). 1324static void adjustForLTGFR(Comparison &C) { 1325 // Check for a comparison between (shl X, 32) and 0. 1326 if (C.Op0.getOpcode() == ISD::SHL && 1327 C.Op0.getValueType() == MVT::i64 && 1328 C.Op1.getOpcode() == ISD::Constant && 1329 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1330 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1)); 1331 if (C1 && C1->getZExtValue() == 32) { 1332 SDValue ShlOp0 = C.Op0.getOperand(0); 1333 // See whether X has any SIGN_EXTEND_INREG uses. 1334 for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) { 1335 SDNode *N = *I; 1336 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG && 1337 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) { 1338 C.Op0 = SDValue(N, 0); 1339 return; 1340 } 1341 } 1342 } 1343 } 1344} 1345 1346// If C compares the truncation of an extending load, try to compare 1347// the untruncated value instead. This exposes more opportunities to 1348// reuse CC. 1349static void adjustICmpTruncate(SelectionDAG &DAG, Comparison &C) { 1350 if (C.Op0.getOpcode() == ISD::TRUNCATE && 1351 C.Op0.getOperand(0).getOpcode() == ISD::LOAD && 1352 C.Op1.getOpcode() == ISD::Constant && 1353 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1354 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0)); 1355 if (L->getMemoryVT().getStoreSizeInBits() 1356 <= C.Op0.getValueType().getSizeInBits()) { 1357 unsigned Type = L->getExtensionType(); 1358 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) || 1359 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) { 1360 C.Op0 = C.Op0.getOperand(0); 1361 C.Op1 = DAG.getConstant(0, C.Op0.getValueType()); 1362 } 1363 } 1364 } 1365} 1366 1367// Return true if shift operation N has an in-range constant shift value. 1368// Store it in ShiftVal if so. 1369static bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 1370 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1371 if (!Shift) 1372 return false; 1373 1374 uint64_t Amount = Shift->getZExtValue(); 1375 if (Amount >= N.getValueType().getSizeInBits()) 1376 return false; 1377 1378 ShiftVal = Amount; 1379 return true; 1380} 1381 1382// Check whether an AND with Mask is suitable for a TEST UNDER MASK 1383// instruction and whether the CC value is descriptive enough to handle 1384// a comparison of type Opcode between the AND result and CmpVal. 1385// CCMask says which comparison result is being tested and BitSize is 1386// the number of bits in the operands. If TEST UNDER MASK can be used, 1387// return the corresponding CC mask, otherwise return 0. 1388static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 1389 uint64_t Mask, uint64_t CmpVal, 1390 unsigned ICmpType) { 1391 assert(Mask != 0 && "ANDs with zero should have been removed by now"); 1392 1393 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 1394 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 1395 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 1396 return 0; 1397 1398 // Work out the masks for the lowest and highest bits. 1399 unsigned HighShift = 63 - countLeadingZeros(Mask); 1400 uint64_t High = uint64_t(1) << HighShift; 1401 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 1402 1403 // Signed ordered comparisons are effectively unsigned if the sign 1404 // bit is dropped. 1405 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 1406 1407 // Check for equality comparisons with 0, or the equivalent. 1408 if (CmpVal == 0) { 1409 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1410 return SystemZ::CCMASK_TM_ALL_0; 1411 if (CCMask == SystemZ::CCMASK_CMP_NE) 1412 return SystemZ::CCMASK_TM_SOME_1; 1413 } 1414 if (EffectivelyUnsigned && CmpVal <= Low) { 1415 if (CCMask == SystemZ::CCMASK_CMP_LT) 1416 return SystemZ::CCMASK_TM_ALL_0; 1417 if (CCMask == SystemZ::CCMASK_CMP_GE) 1418 return SystemZ::CCMASK_TM_SOME_1; 1419 } 1420 if (EffectivelyUnsigned && CmpVal < Low) { 1421 if (CCMask == SystemZ::CCMASK_CMP_LE) 1422 return SystemZ::CCMASK_TM_ALL_0; 1423 if (CCMask == SystemZ::CCMASK_CMP_GT) 1424 return SystemZ::CCMASK_TM_SOME_1; 1425 } 1426 1427 // Check for equality comparisons with the mask, or the equivalent. 1428 if (CmpVal == Mask) { 1429 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1430 return SystemZ::CCMASK_TM_ALL_1; 1431 if (CCMask == SystemZ::CCMASK_CMP_NE) 1432 return SystemZ::CCMASK_TM_SOME_0; 1433 } 1434 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 1435 if (CCMask == SystemZ::CCMASK_CMP_GT) 1436 return SystemZ::CCMASK_TM_ALL_1; 1437 if (CCMask == SystemZ::CCMASK_CMP_LE) 1438 return SystemZ::CCMASK_TM_SOME_0; 1439 } 1440 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 1441 if (CCMask == SystemZ::CCMASK_CMP_GE) 1442 return SystemZ::CCMASK_TM_ALL_1; 1443 if (CCMask == SystemZ::CCMASK_CMP_LT) 1444 return SystemZ::CCMASK_TM_SOME_0; 1445 } 1446 1447 // Check for ordered comparisons with the top bit. 1448 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 1449 if (CCMask == SystemZ::CCMASK_CMP_LE) 1450 return SystemZ::CCMASK_TM_MSB_0; 1451 if (CCMask == SystemZ::CCMASK_CMP_GT) 1452 return SystemZ::CCMASK_TM_MSB_1; 1453 } 1454 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 1455 if (CCMask == SystemZ::CCMASK_CMP_LT) 1456 return SystemZ::CCMASK_TM_MSB_0; 1457 if (CCMask == SystemZ::CCMASK_CMP_GE) 1458 return SystemZ::CCMASK_TM_MSB_1; 1459 } 1460 1461 // If there are just two bits, we can do equality checks for Low and High 1462 // as well. 1463 if (Mask == Low + High) { 1464 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 1465 return SystemZ::CCMASK_TM_MIXED_MSB_0; 1466 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 1467 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 1468 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 1469 return SystemZ::CCMASK_TM_MIXED_MSB_1; 1470 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 1471 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 1472 } 1473 1474 // Looks like we've exhausted our options. 1475 return 0; 1476} 1477 1478// See whether C can be implemented as a TEST UNDER MASK instruction. 1479// Update the arguments with the TM version if so. 1480static void adjustForTestUnderMask(SelectionDAG &DAG, Comparison &C) { 1481 // Check that we have a comparison with a constant. 1482 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 1483 if (!ConstOp1) 1484 return; 1485 uint64_t CmpVal = ConstOp1->getZExtValue(); 1486 1487 // Check whether the nonconstant input is an AND with a constant mask. 1488 Comparison NewC(C); 1489 uint64_t MaskVal; 1490 ConstantSDNode *Mask = nullptr; 1491 if (C.Op0.getOpcode() == ISD::AND) { 1492 NewC.Op0 = C.Op0.getOperand(0); 1493 NewC.Op1 = C.Op0.getOperand(1); 1494 Mask = dyn_cast<ConstantSDNode>(NewC.Op1); 1495 if (!Mask) 1496 return; 1497 MaskVal = Mask->getZExtValue(); 1498 } else { 1499 // There is no instruction to compare with a 64-bit immediate 1500 // so use TMHH instead if possible. We need an unsigned ordered 1501 // comparison with an i64 immediate. 1502 if (NewC.Op0.getValueType() != MVT::i64 || 1503 NewC.CCMask == SystemZ::CCMASK_CMP_EQ || 1504 NewC.CCMask == SystemZ::CCMASK_CMP_NE || 1505 NewC.ICmpType == SystemZICMP::SignedOnly) 1506 return; 1507 // Convert LE and GT comparisons into LT and GE. 1508 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE || 1509 NewC.CCMask == SystemZ::CCMASK_CMP_GT) { 1510 if (CmpVal == uint64_t(-1)) 1511 return; 1512 CmpVal += 1; 1513 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ; 1514 } 1515 // If the low N bits of Op1 are zero than the low N bits of Op0 can 1516 // be masked off without changing the result. 1517 MaskVal = -(CmpVal & -CmpVal); 1518 NewC.ICmpType = SystemZICMP::UnsignedOnly; 1519 } 1520 1521 // Check whether the combination of mask, comparison value and comparison 1522 // type are suitable. 1523 unsigned BitSize = NewC.Op0.getValueType().getSizeInBits(); 1524 unsigned NewCCMask, ShiftVal; 1525 if (NewC.ICmpType != SystemZICMP::SignedOnly && 1526 NewC.Op0.getOpcode() == ISD::SHL && 1527 isSimpleShift(NewC.Op0, ShiftVal) && 1528 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 1529 MaskVal >> ShiftVal, 1530 CmpVal >> ShiftVal, 1531 SystemZICMP::Any))) { 1532 NewC.Op0 = NewC.Op0.getOperand(0); 1533 MaskVal >>= ShiftVal; 1534 } else if (NewC.ICmpType != SystemZICMP::SignedOnly && 1535 NewC.Op0.getOpcode() == ISD::SRL && 1536 isSimpleShift(NewC.Op0, ShiftVal) && 1537 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 1538 MaskVal << ShiftVal, 1539 CmpVal << ShiftVal, 1540 SystemZICMP::UnsignedOnly))) { 1541 NewC.Op0 = NewC.Op0.getOperand(0); 1542 MaskVal <<= ShiftVal; 1543 } else { 1544 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal, 1545 NewC.ICmpType); 1546 if (!NewCCMask) 1547 return; 1548 } 1549 1550 // Go ahead and make the change. 1551 C.Opcode = SystemZISD::TM; 1552 C.Op0 = NewC.Op0; 1553 if (Mask && Mask->getZExtValue() == MaskVal) 1554 C.Op1 = SDValue(Mask, 0); 1555 else 1556 C.Op1 = DAG.getConstant(MaskVal, C.Op0.getValueType()); 1557 C.CCValid = SystemZ::CCMASK_TM; 1558 C.CCMask = NewCCMask; 1559} 1560 1561// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1. 1562static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 1563 ISD::CondCode Cond) { 1564 Comparison C(CmpOp0, CmpOp1); 1565 C.CCMask = CCMaskForCondCode(Cond); 1566 if (C.Op0.getValueType().isFloatingPoint()) { 1567 C.CCValid = SystemZ::CCMASK_FCMP; 1568 C.Opcode = SystemZISD::FCMP; 1569 adjustForFNeg(C); 1570 } else { 1571 C.CCValid = SystemZ::CCMASK_ICMP; 1572 C.Opcode = SystemZISD::ICMP; 1573 // Choose the type of comparison. Equality and inequality tests can 1574 // use either signed or unsigned comparisons. The choice also doesn't 1575 // matter if both sign bits are known to be clear. In those cases we 1576 // want to give the main isel code the freedom to choose whichever 1577 // form fits best. 1578 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 1579 C.CCMask == SystemZ::CCMASK_CMP_NE || 1580 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1))) 1581 C.ICmpType = SystemZICMP::Any; 1582 else if (C.CCMask & SystemZ::CCMASK_CMP_UO) 1583 C.ICmpType = SystemZICMP::UnsignedOnly; 1584 else 1585 C.ICmpType = SystemZICMP::SignedOnly; 1586 C.CCMask &= ~SystemZ::CCMASK_CMP_UO; 1587 adjustZeroCmp(DAG, C); 1588 adjustSubwordCmp(DAG, C); 1589 adjustForSubtraction(DAG, C); 1590 adjustForLTGFR(C); 1591 adjustICmpTruncate(DAG, C); 1592 } 1593 1594 if (shouldSwapCmpOperands(C)) { 1595 std::swap(C.Op0, C.Op1); 1596 C.CCMask = reverseCCMask(C.CCMask); 1597 } 1598 1599 adjustForTestUnderMask(DAG, C); 1600 return C; 1601} 1602 1603// Emit the comparison instruction described by C. 1604static SDValue emitCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) { 1605 if (C.Opcode == SystemZISD::ICMP) 1606 return DAG.getNode(SystemZISD::ICMP, DL, MVT::Glue, C.Op0, C.Op1, 1607 DAG.getConstant(C.ICmpType, MVT::i32)); 1608 if (C.Opcode == SystemZISD::TM) { 1609 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 1610 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 1611 return DAG.getNode(SystemZISD::TM, DL, MVT::Glue, C.Op0, C.Op1, 1612 DAG.getConstant(RegisterOnly, MVT::i32)); 1613 } 1614 return DAG.getNode(C.Opcode, DL, MVT::Glue, C.Op0, C.Op1); 1615} 1616 1617// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1618// 64 bits. Extend is the extension type to use. Store the high part 1619// in Hi and the low part in Lo. 1620static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1621 unsigned Extend, SDValue Op0, SDValue Op1, 1622 SDValue &Hi, SDValue &Lo) { 1623 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1624 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1625 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1626 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1627 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1628 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1629} 1630 1631// Lower a binary operation that produces two VT results, one in each 1632// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1633// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1634// on the extended Op0 and (unextended) Op1. Store the even register result 1635// in Even and the odd register result in Odd. 1636static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1637 unsigned Extend, unsigned Opcode, 1638 SDValue Op0, SDValue Op1, 1639 SDValue &Even, SDValue &Odd) { 1640 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1641 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1642 SDValue(In128, 0), Op1); 1643 bool Is32Bit = is32Bit(VT); 1644 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 1645 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 1646} 1647 1648// Return an i32 value that is 1 if the CC value produced by Glue is 1649// in the mask CCMask and 0 otherwise. CC is known to have a value 1650// in CCValid, so other values can be ignored. 1651static SDValue emitSETCC(SelectionDAG &DAG, SDLoc DL, SDValue Glue, 1652 unsigned CCValid, unsigned CCMask) { 1653 IPMConversion Conversion = getIPMConversion(CCValid, CCMask); 1654 SDValue Result = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue); 1655 1656 if (Conversion.XORValue) 1657 Result = DAG.getNode(ISD::XOR, DL, MVT::i32, Result, 1658 DAG.getConstant(Conversion.XORValue, MVT::i32)); 1659 1660 if (Conversion.AddValue) 1661 Result = DAG.getNode(ISD::ADD, DL, MVT::i32, Result, 1662 DAG.getConstant(Conversion.AddValue, MVT::i32)); 1663 1664 // The SHR/AND sequence should get optimized to an RISBG. 1665 Result = DAG.getNode(ISD::SRL, DL, MVT::i32, Result, 1666 DAG.getConstant(Conversion.Bit, MVT::i32)); 1667 if (Conversion.Bit != 31) 1668 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, 1669 DAG.getConstant(1, MVT::i32)); 1670 return Result; 1671} 1672 1673SDValue SystemZTargetLowering::lowerSETCC(SDValue Op, 1674 SelectionDAG &DAG) const { 1675 SDValue CmpOp0 = Op.getOperand(0); 1676 SDValue CmpOp1 = Op.getOperand(1); 1677 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 1678 SDLoc DL(Op); 1679 1680 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1681 SDValue Glue = emitCmp(DAG, DL, C); 1682 return emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 1683} 1684 1685SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1686 SDValue Chain = Op.getOperand(0); 1687 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1688 SDValue CmpOp0 = Op.getOperand(2); 1689 SDValue CmpOp1 = Op.getOperand(3); 1690 SDValue Dest = Op.getOperand(4); 1691 SDLoc DL(Op); 1692 1693 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1694 SDValue Glue = emitCmp(DAG, DL, C); 1695 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1696 Chain, DAG.getConstant(C.CCValid, MVT::i32), 1697 DAG.getConstant(C.CCMask, MVT::i32), Dest, Glue); 1698} 1699 1700// Return true if Pos is CmpOp and Neg is the negative of CmpOp, 1701// allowing Pos and Neg to be wider than CmpOp. 1702static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) { 1703 return (Neg.getOpcode() == ISD::SUB && 1704 Neg.getOperand(0).getOpcode() == ISD::Constant && 1705 cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 && 1706 Neg.getOperand(1) == Pos && 1707 (Pos == CmpOp || 1708 (Pos.getOpcode() == ISD::SIGN_EXTEND && 1709 Pos.getOperand(0) == CmpOp))); 1710} 1711 1712// Return the absolute or negative absolute of Op; IsNegative decides which. 1713static SDValue getAbsolute(SelectionDAG &DAG, SDLoc DL, SDValue Op, 1714 bool IsNegative) { 1715 Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op); 1716 if (IsNegative) 1717 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(), 1718 DAG.getConstant(0, Op.getValueType()), Op); 1719 return Op; 1720} 1721 1722SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1723 SelectionDAG &DAG) const { 1724 SDValue CmpOp0 = Op.getOperand(0); 1725 SDValue CmpOp1 = Op.getOperand(1); 1726 SDValue TrueOp = Op.getOperand(2); 1727 SDValue FalseOp = Op.getOperand(3); 1728 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1729 SDLoc DL(Op); 1730 1731 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1732 1733 // Check for absolute and negative-absolute selections, including those 1734 // where the comparison value is sign-extended (for LPGFR and LNGFR). 1735 // This check supplements the one in DAGCombiner. 1736 if (C.Opcode == SystemZISD::ICMP && 1737 C.CCMask != SystemZ::CCMASK_CMP_EQ && 1738 C.CCMask != SystemZ::CCMASK_CMP_NE && 1739 C.Op1.getOpcode() == ISD::Constant && 1740 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1741 if (isAbsolute(C.Op0, TrueOp, FalseOp)) 1742 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT); 1743 if (isAbsolute(C.Op0, FalseOp, TrueOp)) 1744 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT); 1745 } 1746 1747 SDValue Glue = emitCmp(DAG, DL, C); 1748 1749 // Special case for handling -1/0 results. The shifts we use here 1750 // should get optimized with the IPM conversion sequence. 1751 auto *TrueC = dyn_cast<ConstantSDNode>(TrueOp); 1752 auto *FalseC = dyn_cast<ConstantSDNode>(FalseOp); 1753 if (TrueC && FalseC) { 1754 int64_t TrueVal = TrueC->getSExtValue(); 1755 int64_t FalseVal = FalseC->getSExtValue(); 1756 if ((TrueVal == -1 && FalseVal == 0) || (TrueVal == 0 && FalseVal == -1)) { 1757 // Invert the condition if we want -1 on false. 1758 if (TrueVal == 0) 1759 C.CCMask ^= C.CCValid; 1760 SDValue Result = emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 1761 EVT VT = Op.getValueType(); 1762 // Extend the result to VT. Upper bits are ignored. 1763 if (!is32Bit(VT)) 1764 Result = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Result); 1765 // Sign-extend from the low bit. 1766 SDValue ShAmt = DAG.getConstant(VT.getSizeInBits() - 1, MVT::i32); 1767 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Result, ShAmt); 1768 return DAG.getNode(ISD::SRA, DL, VT, Shl, ShAmt); 1769 } 1770 } 1771 1772 SmallVector<SDValue, 5> Ops; 1773 Ops.push_back(TrueOp); 1774 Ops.push_back(FalseOp); 1775 Ops.push_back(DAG.getConstant(C.CCValid, MVT::i32)); 1776 Ops.push_back(DAG.getConstant(C.CCMask, MVT::i32)); 1777 Ops.push_back(Glue); 1778 1779 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1780 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, Ops); 1781} 1782 1783SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1784 SelectionDAG &DAG) const { 1785 SDLoc DL(Node); 1786 const GlobalValue *GV = Node->getGlobal(); 1787 int64_t Offset = Node->getOffset(); 1788 EVT PtrVT = getPointerTy(); 1789 Reloc::Model RM = TM.getRelocationModel(); 1790 CodeModel::Model CM = TM.getCodeModel(); 1791 1792 SDValue Result; 1793 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1794 // Assign anchors at 1<<12 byte boundaries. 1795 uint64_t Anchor = Offset & ~uint64_t(0xfff); 1796 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 1797 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1798 1799 // The offset can be folded into the address if it is aligned to a halfword. 1800 Offset -= Anchor; 1801 if (Offset != 0 && (Offset & 1) == 0) { 1802 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 1803 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 1804 Offset = 0; 1805 } 1806 } else { 1807 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1808 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1809 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1810 MachinePointerInfo::getGOT(), false, false, false, 0); 1811 } 1812 1813 // If there was a non-zero offset that we didn't fold, create an explicit 1814 // addition for it. 1815 if (Offset != 0) 1816 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1817 DAG.getConstant(Offset, PtrVT)); 1818 1819 return Result; 1820} 1821 1822SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1823 SelectionDAG &DAG) const { 1824 SDLoc DL(Node); 1825 const GlobalValue *GV = Node->getGlobal(); 1826 EVT PtrVT = getPointerTy(); 1827 TLSModel::Model model = TM.getTLSModel(GV); 1828 1829 if (model != TLSModel::LocalExec) 1830 llvm_unreachable("only local-exec TLS mode supported"); 1831 1832 // The high part of the thread pointer is in access register 0. 1833 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1834 DAG.getConstant(0, MVT::i32)); 1835 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1836 1837 // The low part of the thread pointer is in access register 1. 1838 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1839 DAG.getConstant(1, MVT::i32)); 1840 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1841 1842 // Merge them into a single 64-bit address. 1843 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1844 DAG.getConstant(32, PtrVT)); 1845 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1846 1847 // Get the offset of GA from the thread pointer. 1848 SystemZConstantPoolValue *CPV = 1849 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1850 1851 // Force the offset into the constant pool and load it from there. 1852 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1853 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1854 CPAddr, MachinePointerInfo::getConstantPool(), 1855 false, false, false, 0); 1856 1857 // Add the base and offset together. 1858 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1859} 1860 1861SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1862 SelectionDAG &DAG) const { 1863 SDLoc DL(Node); 1864 const BlockAddress *BA = Node->getBlockAddress(); 1865 int64_t Offset = Node->getOffset(); 1866 EVT PtrVT = getPointerTy(); 1867 1868 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1869 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1870 return Result; 1871} 1872 1873SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1874 SelectionDAG &DAG) const { 1875 SDLoc DL(JT); 1876 EVT PtrVT = getPointerTy(); 1877 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1878 1879 // Use LARL to load the address of the table. 1880 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1881} 1882 1883SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1884 SelectionDAG &DAG) const { 1885 SDLoc DL(CP); 1886 EVT PtrVT = getPointerTy(); 1887 1888 SDValue Result; 1889 if (CP->isMachineConstantPoolEntry()) 1890 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1891 CP->getAlignment()); 1892 else 1893 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1894 CP->getAlignment(), CP->getOffset()); 1895 1896 // Use LARL to load the address of the constant pool entry. 1897 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1898} 1899 1900SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1901 SelectionDAG &DAG) const { 1902 SDLoc DL(Op); 1903 SDValue In = Op.getOperand(0); 1904 EVT InVT = In.getValueType(); 1905 EVT ResVT = Op.getValueType(); 1906 1907 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1908 SDValue In64; 1909 if (Subtarget.hasHighWord()) { 1910 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, 1911 MVT::i64); 1912 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1913 MVT::i64, SDValue(U64, 0), In); 1914 } else { 1915 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1916 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, 1917 DAG.getConstant(32, MVT::i64)); 1918 } 1919 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); 1920 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, 1921 DL, MVT::f32, Out64); 1922 } 1923 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1924 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1925 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1926 MVT::f64, SDValue(U64, 0), In); 1927 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 1928 if (Subtarget.hasHighWord()) 1929 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, 1930 MVT::i32, Out64); 1931 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, 1932 DAG.getConstant(32, MVT::i64)); 1933 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1934 } 1935 llvm_unreachable("Unexpected bitcast combination"); 1936} 1937 1938SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1939 SelectionDAG &DAG) const { 1940 MachineFunction &MF = DAG.getMachineFunction(); 1941 SystemZMachineFunctionInfo *FuncInfo = 1942 MF.getInfo<SystemZMachineFunctionInfo>(); 1943 EVT PtrVT = getPointerTy(); 1944 1945 SDValue Chain = Op.getOperand(0); 1946 SDValue Addr = Op.getOperand(1); 1947 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1948 SDLoc DL(Op); 1949 1950 // The initial values of each field. 1951 const unsigned NumFields = 4; 1952 SDValue Fields[NumFields] = { 1953 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1954 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1955 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1956 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1957 }; 1958 1959 // Store each field into its respective slot. 1960 SDValue MemOps[NumFields]; 1961 unsigned Offset = 0; 1962 for (unsigned I = 0; I < NumFields; ++I) { 1963 SDValue FieldAddr = Addr; 1964 if (Offset != 0) 1965 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1966 DAG.getIntPtrConstant(Offset)); 1967 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1968 MachinePointerInfo(SV, Offset), 1969 false, false, 0); 1970 Offset += 8; 1971 } 1972 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 1973} 1974 1975SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1976 SelectionDAG &DAG) const { 1977 SDValue Chain = Op.getOperand(0); 1978 SDValue DstPtr = Op.getOperand(1); 1979 SDValue SrcPtr = Op.getOperand(2); 1980 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1981 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1982 SDLoc DL(Op); 1983 1984 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1985 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1986 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1987} 1988 1989SDValue SystemZTargetLowering:: 1990lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1991 SDValue Chain = Op.getOperand(0); 1992 SDValue Size = Op.getOperand(1); 1993 SDLoc DL(Op); 1994 1995 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1996 1997 // Get a reference to the stack pointer. 1998 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1999 2000 // Get the new stack pointer value. 2001 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 2002 2003 // Copy the new stack pointer back. 2004 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 2005 2006 // The allocated data lives above the 160 bytes allocated for the standard 2007 // frame, plus any outgoing stack arguments. We don't know how much that 2008 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 2009 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 2010 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 2011 2012 SDValue Ops[2] = { Result, Chain }; 2013 return DAG.getMergeValues(Ops, DL); 2014} 2015 2016SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 2017 SelectionDAG &DAG) const { 2018 EVT VT = Op.getValueType(); 2019 SDLoc DL(Op); 2020 SDValue Ops[2]; 2021 if (is32Bit(VT)) 2022 // Just do a normal 64-bit multiplication and extract the results. 2023 // We define this so that it can be used for constant division. 2024 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 2025 Op.getOperand(1), Ops[1], Ops[0]); 2026 else { 2027 // Do a full 128-bit multiplication based on UMUL_LOHI64: 2028 // 2029 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 2030 // 2031 // but using the fact that the upper halves are either all zeros 2032 // or all ones: 2033 // 2034 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 2035 // 2036 // and grouping the right terms together since they are quicker than the 2037 // multiplication: 2038 // 2039 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 2040 SDValue C63 = DAG.getConstant(63, MVT::i64); 2041 SDValue LL = Op.getOperand(0); 2042 SDValue RL = Op.getOperand(1); 2043 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 2044 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 2045 // UMUL_LOHI64 returns the low result in the odd register and the high 2046 // result in the even register. SMUL_LOHI is defined to return the 2047 // low half first, so the results are in reverse order. 2048 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2049 LL, RL, Ops[1], Ops[0]); 2050 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 2051 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 2052 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 2053 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 2054 } 2055 return DAG.getMergeValues(Ops, DL); 2056} 2057 2058SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 2059 SelectionDAG &DAG) const { 2060 EVT VT = Op.getValueType(); 2061 SDLoc DL(Op); 2062 SDValue Ops[2]; 2063 if (is32Bit(VT)) 2064 // Just do a normal 64-bit multiplication and extract the results. 2065 // We define this so that it can be used for constant division. 2066 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 2067 Op.getOperand(1), Ops[1], Ops[0]); 2068 else 2069 // UMUL_LOHI64 returns the low result in the odd register and the high 2070 // result in the even register. UMUL_LOHI is defined to return the 2071 // low half first, so the results are in reverse order. 2072 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2073 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2074 return DAG.getMergeValues(Ops, DL); 2075} 2076 2077SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 2078 SelectionDAG &DAG) const { 2079 SDValue Op0 = Op.getOperand(0); 2080 SDValue Op1 = Op.getOperand(1); 2081 EVT VT = Op.getValueType(); 2082 SDLoc DL(Op); 2083 unsigned Opcode; 2084 2085 // We use DSGF for 32-bit division. 2086 if (is32Bit(VT)) { 2087 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 2088 Opcode = SystemZISD::SDIVREM32; 2089 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 2090 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 2091 Opcode = SystemZISD::SDIVREM32; 2092 } else 2093 Opcode = SystemZISD::SDIVREM64; 2094 2095 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 2096 // input is "don't care". The instruction returns the remainder in 2097 // the even register and the quotient in the odd register. 2098 SDValue Ops[2]; 2099 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 2100 Op0, Op1, Ops[1], Ops[0]); 2101 return DAG.getMergeValues(Ops, DL); 2102} 2103 2104SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 2105 SelectionDAG &DAG) const { 2106 EVT VT = Op.getValueType(); 2107 SDLoc DL(Op); 2108 2109 // DL(G) uses a double-width dividend, so we need to clear the even 2110 // register in the GR128 input. The instruction returns the remainder 2111 // in the even register and the quotient in the odd register. 2112 SDValue Ops[2]; 2113 if (is32Bit(VT)) 2114 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 2115 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2116 else 2117 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 2118 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2119 return DAG.getMergeValues(Ops, DL); 2120} 2121 2122SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 2123 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 2124 2125 // Get the known-zero masks for each operand. 2126 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 2127 APInt KnownZero[2], KnownOne[2]; 2128 DAG.computeKnownBits(Ops[0], KnownZero[0], KnownOne[0]); 2129 DAG.computeKnownBits(Ops[1], KnownZero[1], KnownOne[1]); 2130 2131 // See if the upper 32 bits of one operand and the lower 32 bits of the 2132 // other are known zero. They are the low and high operands respectively. 2133 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 2134 KnownZero[1].getZExtValue() }; 2135 unsigned High, Low; 2136 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 2137 High = 1, Low = 0; 2138 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 2139 High = 0, Low = 1; 2140 else 2141 return Op; 2142 2143 SDValue LowOp = Ops[Low]; 2144 SDValue HighOp = Ops[High]; 2145 2146 // If the high part is a constant, we're better off using IILH. 2147 if (HighOp.getOpcode() == ISD::Constant) 2148 return Op; 2149 2150 // If the low part is a constant that is outside the range of LHI, 2151 // then we're better off using IILF. 2152 if (LowOp.getOpcode() == ISD::Constant) { 2153 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 2154 if (!isInt<16>(Value)) 2155 return Op; 2156 } 2157 2158 // Check whether the high part is an AND that doesn't change the 2159 // high 32 bits and just masks out low bits. We can skip it if so. 2160 if (HighOp.getOpcode() == ISD::AND && 2161 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 2162 SDValue HighOp0 = HighOp.getOperand(0); 2163 uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue(); 2164 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff)))) 2165 HighOp = HighOp0; 2166 } 2167 2168 // Take advantage of the fact that all GR32 operations only change the 2169 // low 32 bits by truncating Low to an i32 and inserting it directly 2170 // using a subreg. The interesting cases are those where the truncation 2171 // can be folded. 2172 SDLoc DL(Op); 2173 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 2174 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, 2175 MVT::i64, HighOp, Low32); 2176} 2177 2178// Op is an atomic load. Lower it into a normal volatile load. 2179SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 2180 SelectionDAG &DAG) const { 2181 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2182 return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(), 2183 Node->getChain(), Node->getBasePtr(), 2184 Node->getMemoryVT(), Node->getMemOperand()); 2185} 2186 2187// Op is an atomic store. Lower it into a normal volatile store followed 2188// by a serialization. 2189SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op, 2190 SelectionDAG &DAG) const { 2191 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2192 SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(), 2193 Node->getBasePtr(), Node->getMemoryVT(), 2194 Node->getMemOperand()); 2195 return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other, 2196 Chain), 0); 2197} 2198 2199// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 2200// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 2201SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, 2202 SelectionDAG &DAG, 2203 unsigned Opcode) const { 2204 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2205 2206 // 32-bit operations need no code outside the main loop. 2207 EVT NarrowVT = Node->getMemoryVT(); 2208 EVT WideVT = MVT::i32; 2209 if (NarrowVT == WideVT) 2210 return Op; 2211 2212 int64_t BitSize = NarrowVT.getSizeInBits(); 2213 SDValue ChainIn = Node->getChain(); 2214 SDValue Addr = Node->getBasePtr(); 2215 SDValue Src2 = Node->getVal(); 2216 MachineMemOperand *MMO = Node->getMemOperand(); 2217 SDLoc DL(Node); 2218 EVT PtrVT = Addr.getValueType(); 2219 2220 // Convert atomic subtracts of constants into additions. 2221 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 2222 if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) { 2223 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 2224 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 2225 } 2226 2227 // Get the address of the containing word. 2228 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 2229 DAG.getConstant(-4, PtrVT)); 2230 2231 // Get the number of bits that the word must be rotated left in order 2232 // to bring the field to the top bits of a GR32. 2233 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 2234 DAG.getConstant(3, PtrVT)); 2235 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 2236 2237 // Get the complementing shift amount, for rotating a field in the top 2238 // bits back to its proper position. 2239 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 2240 DAG.getConstant(0, WideVT), BitShift); 2241 2242 // Extend the source operand to 32 bits and prepare it for the inner loop. 2243 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 2244 // operations require the source to be shifted in advance. (This shift 2245 // can be folded if the source is constant.) For AND and NAND, the lower 2246 // bits must be set, while for other opcodes they should be left clear. 2247 if (Opcode != SystemZISD::ATOMIC_SWAPW) 2248 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 2249 DAG.getConstant(32 - BitSize, WideVT)); 2250 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 2251 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 2252 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 2253 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 2254 2255 // Construct the ATOMIC_LOADW_* node. 2256 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2257 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 2258 DAG.getConstant(BitSize, WideVT) }; 2259 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 2260 NarrowVT, MMO); 2261 2262 // Rotate the result of the final CS so that the field is in the lower 2263 // bits of a GR32, then truncate it. 2264 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 2265 DAG.getConstant(BitSize, WideVT)); 2266 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 2267 2268 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 2269 return DAG.getMergeValues(RetOps, DL); 2270} 2271 2272// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations 2273// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit 2274// operations into additions. 2275SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op, 2276 SelectionDAG &DAG) const { 2277 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2278 EVT MemVT = Node->getMemoryVT(); 2279 if (MemVT == MVT::i32 || MemVT == MVT::i64) { 2280 // A full-width operation. 2281 assert(Op.getValueType() == MemVT && "Mismatched VTs"); 2282 SDValue Src2 = Node->getVal(); 2283 SDValue NegSrc2; 2284 SDLoc DL(Src2); 2285 2286 if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) { 2287 // Use an addition if the operand is constant and either LAA(G) is 2288 // available or the negative value is in the range of A(G)FHI. 2289 int64_t Value = (-Op2->getAPIntValue()).getSExtValue(); 2290 if (isInt<32>(Value) || TM.getSubtargetImpl()->hasInterlockedAccess1()) 2291 NegSrc2 = DAG.getConstant(Value, MemVT); 2292 } else if (TM.getSubtargetImpl()->hasInterlockedAccess1()) 2293 // Use LAA(G) if available. 2294 NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, MemVT), 2295 Src2); 2296 2297 if (NegSrc2.getNode()) 2298 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, 2299 Node->getChain(), Node->getBasePtr(), NegSrc2, 2300 Node->getMemOperand(), Node->getOrdering(), 2301 Node->getSynchScope()); 2302 2303 // Use the node as-is. 2304 return Op; 2305 } 2306 2307 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 2308} 2309 2310// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 2311// into a fullword ATOMIC_CMP_SWAPW operation. 2312SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 2313 SelectionDAG &DAG) const { 2314 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2315 2316 // We have native support for 32-bit compare and swap. 2317 EVT NarrowVT = Node->getMemoryVT(); 2318 EVT WideVT = MVT::i32; 2319 if (NarrowVT == WideVT) 2320 return Op; 2321 2322 int64_t BitSize = NarrowVT.getSizeInBits(); 2323 SDValue ChainIn = Node->getOperand(0); 2324 SDValue Addr = Node->getOperand(1); 2325 SDValue CmpVal = Node->getOperand(2); 2326 SDValue SwapVal = Node->getOperand(3); 2327 MachineMemOperand *MMO = Node->getMemOperand(); 2328 SDLoc DL(Node); 2329 EVT PtrVT = Addr.getValueType(); 2330 2331 // Get the address of the containing word. 2332 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 2333 DAG.getConstant(-4, PtrVT)); 2334 2335 // Get the number of bits that the word must be rotated left in order 2336 // to bring the field to the top bits of a GR32. 2337 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 2338 DAG.getConstant(3, PtrVT)); 2339 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 2340 2341 // Get the complementing shift amount, for rotating a field in the top 2342 // bits back to its proper position. 2343 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 2344 DAG.getConstant(0, WideVT), BitShift); 2345 2346 // Construct the ATOMIC_CMP_SWAPW node. 2347 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2348 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 2349 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 2350 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 2351 VTList, Ops, NarrowVT, MMO); 2352 return AtomicOp; 2353} 2354 2355SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 2356 SelectionDAG &DAG) const { 2357 MachineFunction &MF = DAG.getMachineFunction(); 2358 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2359 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 2360 SystemZ::R15D, Op.getValueType()); 2361} 2362 2363SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 2364 SelectionDAG &DAG) const { 2365 MachineFunction &MF = DAG.getMachineFunction(); 2366 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2367 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 2368 SystemZ::R15D, Op.getOperand(1)); 2369} 2370 2371SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op, 2372 SelectionDAG &DAG) const { 2373 bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 2374 if (!IsData) 2375 // Just preserve the chain. 2376 return Op.getOperand(0); 2377 2378 bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 2379 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ; 2380 auto *Node = cast<MemIntrinsicSDNode>(Op.getNode()); 2381 SDValue Ops[] = { 2382 Op.getOperand(0), 2383 DAG.getConstant(Code, MVT::i32), 2384 Op.getOperand(1) 2385 }; 2386 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, SDLoc(Op), 2387 Node->getVTList(), Ops, 2388 Node->getMemoryVT(), Node->getMemOperand()); 2389} 2390 2391SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 2392 SelectionDAG &DAG) const { 2393 switch (Op.getOpcode()) { 2394 case ISD::BR_CC: 2395 return lowerBR_CC(Op, DAG); 2396 case ISD::SELECT_CC: 2397 return lowerSELECT_CC(Op, DAG); 2398 case ISD::SETCC: 2399 return lowerSETCC(Op, DAG); 2400 case ISD::GlobalAddress: 2401 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 2402 case ISD::GlobalTLSAddress: 2403 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 2404 case ISD::BlockAddress: 2405 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 2406 case ISD::JumpTable: 2407 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 2408 case ISD::ConstantPool: 2409 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 2410 case ISD::BITCAST: 2411 return lowerBITCAST(Op, DAG); 2412 case ISD::VASTART: 2413 return lowerVASTART(Op, DAG); 2414 case ISD::VACOPY: 2415 return lowerVACOPY(Op, DAG); 2416 case ISD::DYNAMIC_STACKALLOC: 2417 return lowerDYNAMIC_STACKALLOC(Op, DAG); 2418 case ISD::SMUL_LOHI: 2419 return lowerSMUL_LOHI(Op, DAG); 2420 case ISD::UMUL_LOHI: 2421 return lowerUMUL_LOHI(Op, DAG); 2422 case ISD::SDIVREM: 2423 return lowerSDIVREM(Op, DAG); 2424 case ISD::UDIVREM: 2425 return lowerUDIVREM(Op, DAG); 2426 case ISD::OR: 2427 return lowerOR(Op, DAG); 2428 case ISD::ATOMIC_SWAP: 2429 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW); 2430 case ISD::ATOMIC_STORE: 2431 return lowerATOMIC_STORE(Op, DAG); 2432 case ISD::ATOMIC_LOAD: 2433 return lowerATOMIC_LOAD(Op, DAG); 2434 case ISD::ATOMIC_LOAD_ADD: 2435 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 2436 case ISD::ATOMIC_LOAD_SUB: 2437 return lowerATOMIC_LOAD_SUB(Op, DAG); 2438 case ISD::ATOMIC_LOAD_AND: 2439 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 2440 case ISD::ATOMIC_LOAD_OR: 2441 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 2442 case ISD::ATOMIC_LOAD_XOR: 2443 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 2444 case ISD::ATOMIC_LOAD_NAND: 2445 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 2446 case ISD::ATOMIC_LOAD_MIN: 2447 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 2448 case ISD::ATOMIC_LOAD_MAX: 2449 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 2450 case ISD::ATOMIC_LOAD_UMIN: 2451 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 2452 case ISD::ATOMIC_LOAD_UMAX: 2453 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 2454 case ISD::ATOMIC_CMP_SWAP: 2455 return lowerATOMIC_CMP_SWAP(Op, DAG); 2456 case ISD::STACKSAVE: 2457 return lowerSTACKSAVE(Op, DAG); 2458 case ISD::STACKRESTORE: 2459 return lowerSTACKRESTORE(Op, DAG); 2460 case ISD::PREFETCH: 2461 return lowerPREFETCH(Op, DAG); 2462 default: 2463 llvm_unreachable("Unexpected node to lower"); 2464 } 2465} 2466 2467const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 2468#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 2469 switch (Opcode) { 2470 OPCODE(RET_FLAG); 2471 OPCODE(CALL); 2472 OPCODE(SIBCALL); 2473 OPCODE(PCREL_WRAPPER); 2474 OPCODE(PCREL_OFFSET); 2475 OPCODE(IABS); 2476 OPCODE(ICMP); 2477 OPCODE(FCMP); 2478 OPCODE(TM); 2479 OPCODE(BR_CCMASK); 2480 OPCODE(SELECT_CCMASK); 2481 OPCODE(ADJDYNALLOC); 2482 OPCODE(EXTRACT_ACCESS); 2483 OPCODE(UMUL_LOHI64); 2484 OPCODE(SDIVREM64); 2485 OPCODE(UDIVREM32); 2486 OPCODE(UDIVREM64); 2487 OPCODE(MVC); 2488 OPCODE(MVC_LOOP); 2489 OPCODE(NC); 2490 OPCODE(NC_LOOP); 2491 OPCODE(OC); 2492 OPCODE(OC_LOOP); 2493 OPCODE(XC); 2494 OPCODE(XC_LOOP); 2495 OPCODE(CLC); 2496 OPCODE(CLC_LOOP); 2497 OPCODE(STRCMP); 2498 OPCODE(STPCPY); 2499 OPCODE(SEARCH_STRING); 2500 OPCODE(IPM); 2501 OPCODE(SERIALIZE); 2502 OPCODE(ATOMIC_SWAPW); 2503 OPCODE(ATOMIC_LOADW_ADD); 2504 OPCODE(ATOMIC_LOADW_SUB); 2505 OPCODE(ATOMIC_LOADW_AND); 2506 OPCODE(ATOMIC_LOADW_OR); 2507 OPCODE(ATOMIC_LOADW_XOR); 2508 OPCODE(ATOMIC_LOADW_NAND); 2509 OPCODE(ATOMIC_LOADW_MIN); 2510 OPCODE(ATOMIC_LOADW_MAX); 2511 OPCODE(ATOMIC_LOADW_UMIN); 2512 OPCODE(ATOMIC_LOADW_UMAX); 2513 OPCODE(ATOMIC_CMP_SWAPW); 2514 OPCODE(PREFETCH); 2515 } 2516 return nullptr; 2517#undef OPCODE 2518} 2519 2520SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, 2521 DAGCombinerInfo &DCI) const { 2522 SelectionDAG &DAG = DCI.DAG; 2523 unsigned Opcode = N->getOpcode(); 2524 if (Opcode == ISD::SIGN_EXTEND) { 2525 // Convert (sext (ashr (shl X, C1), C2)) to 2526 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as 2527 // cheap as narrower ones. 2528 SDValue N0 = N->getOperand(0); 2529 EVT VT = N->getValueType(0); 2530 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) { 2531 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 2532 SDValue Inner = N0.getOperand(0); 2533 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) { 2534 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) { 2535 unsigned Extra = (VT.getSizeInBits() - 2536 N0.getValueType().getSizeInBits()); 2537 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra; 2538 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra; 2539 EVT ShiftVT = N0.getOperand(1).getValueType(); 2540 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT, 2541 Inner.getOperand(0)); 2542 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext, 2543 DAG.getConstant(NewShlAmt, ShiftVT)); 2544 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, 2545 DAG.getConstant(NewSraAmt, ShiftVT)); 2546 } 2547 } 2548 } 2549 } 2550 return SDValue(); 2551} 2552 2553//===----------------------------------------------------------------------===// 2554// Custom insertion 2555//===----------------------------------------------------------------------===// 2556 2557// Create a new basic block after MBB. 2558static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 2559 MachineFunction &MF = *MBB->getParent(); 2560 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 2561 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 2562 return NewMBB; 2563} 2564 2565// Split MBB after MI and return the new block (the one that contains 2566// instructions after MI). 2567static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 2568 MachineBasicBlock *MBB) { 2569 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2570 NewMBB->splice(NewMBB->begin(), MBB, 2571 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 2572 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2573 return NewMBB; 2574} 2575 2576// Split MBB before MI and return the new block (the one that contains MI). 2577static MachineBasicBlock *splitBlockBefore(MachineInstr *MI, 2578 MachineBasicBlock *MBB) { 2579 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2580 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 2581 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2582 return NewMBB; 2583} 2584 2585// Force base value Base into a register before MI. Return the register. 2586static unsigned forceReg(MachineInstr *MI, MachineOperand &Base, 2587 const SystemZInstrInfo *TII) { 2588 if (Base.isReg()) 2589 return Base.getReg(); 2590 2591 MachineBasicBlock *MBB = MI->getParent(); 2592 MachineFunction &MF = *MBB->getParent(); 2593 MachineRegisterInfo &MRI = MF.getRegInfo(); 2594 2595 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2596 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LA), Reg) 2597 .addOperand(Base).addImm(0).addReg(0); 2598 return Reg; 2599} 2600 2601// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 2602MachineBasicBlock * 2603SystemZTargetLowering::emitSelect(MachineInstr *MI, 2604 MachineBasicBlock *MBB) const { 2605 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2606 2607 unsigned DestReg = MI->getOperand(0).getReg(); 2608 unsigned TrueReg = MI->getOperand(1).getReg(); 2609 unsigned FalseReg = MI->getOperand(2).getReg(); 2610 unsigned CCValid = MI->getOperand(3).getImm(); 2611 unsigned CCMask = MI->getOperand(4).getImm(); 2612 DebugLoc DL = MI->getDebugLoc(); 2613 2614 MachineBasicBlock *StartMBB = MBB; 2615 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2616 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2617 2618 // StartMBB: 2619 // BRC CCMask, JoinMBB 2620 // # fallthrough to FalseMBB 2621 MBB = StartMBB; 2622 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2623 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2624 MBB->addSuccessor(JoinMBB); 2625 MBB->addSuccessor(FalseMBB); 2626 2627 // FalseMBB: 2628 // # fallthrough to JoinMBB 2629 MBB = FalseMBB; 2630 MBB->addSuccessor(JoinMBB); 2631 2632 // JoinMBB: 2633 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 2634 // ... 2635 MBB = JoinMBB; 2636 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg) 2637 .addReg(TrueReg).addMBB(StartMBB) 2638 .addReg(FalseReg).addMBB(FalseMBB); 2639 2640 MI->eraseFromParent(); 2641 return JoinMBB; 2642} 2643 2644// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 2645// StoreOpcode is the store to use and Invert says whether the store should 2646// happen when the condition is false rather than true. If a STORE ON 2647// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 2648MachineBasicBlock * 2649SystemZTargetLowering::emitCondStore(MachineInstr *MI, 2650 MachineBasicBlock *MBB, 2651 unsigned StoreOpcode, unsigned STOCOpcode, 2652 bool Invert) const { 2653 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2654 2655 unsigned SrcReg = MI->getOperand(0).getReg(); 2656 MachineOperand Base = MI->getOperand(1); 2657 int64_t Disp = MI->getOperand(2).getImm(); 2658 unsigned IndexReg = MI->getOperand(3).getReg(); 2659 unsigned CCValid = MI->getOperand(4).getImm(); 2660 unsigned CCMask = MI->getOperand(5).getImm(); 2661 DebugLoc DL = MI->getDebugLoc(); 2662 2663 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 2664 2665 // Use STOCOpcode if possible. We could use different store patterns in 2666 // order to avoid matching the index register, but the performance trade-offs 2667 // might be more complicated in that case. 2668 if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) { 2669 if (Invert) 2670 CCMask ^= CCValid; 2671 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 2672 .addReg(SrcReg).addOperand(Base).addImm(Disp) 2673 .addImm(CCValid).addImm(CCMask); 2674 MI->eraseFromParent(); 2675 return MBB; 2676 } 2677 2678 // Get the condition needed to branch around the store. 2679 if (!Invert) 2680 CCMask ^= CCValid; 2681 2682 MachineBasicBlock *StartMBB = MBB; 2683 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2684 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2685 2686 // StartMBB: 2687 // BRC CCMask, JoinMBB 2688 // # fallthrough to FalseMBB 2689 MBB = StartMBB; 2690 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2691 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2692 MBB->addSuccessor(JoinMBB); 2693 MBB->addSuccessor(FalseMBB); 2694 2695 // FalseMBB: 2696 // store %SrcReg, %Disp(%Index,%Base) 2697 // # fallthrough to JoinMBB 2698 MBB = FalseMBB; 2699 BuildMI(MBB, DL, TII->get(StoreOpcode)) 2700 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 2701 MBB->addSuccessor(JoinMBB); 2702 2703 MI->eraseFromParent(); 2704 return JoinMBB; 2705} 2706 2707// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 2708// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 2709// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 2710// BitSize is the width of the field in bits, or 0 if this is a partword 2711// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 2712// is one of the operands. Invert says whether the field should be 2713// inverted after performing BinOpcode (e.g. for NAND). 2714MachineBasicBlock * 2715SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 2716 MachineBasicBlock *MBB, 2717 unsigned BinOpcode, 2718 unsigned BitSize, 2719 bool Invert) const { 2720 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2721 MachineFunction &MF = *MBB->getParent(); 2722 MachineRegisterInfo &MRI = MF.getRegInfo(); 2723 bool IsSubWord = (BitSize < 32); 2724 2725 // Extract the operands. Base can be a register or a frame index. 2726 // Src2 can be a register or immediate. 2727 unsigned Dest = MI->getOperand(0).getReg(); 2728 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2729 int64_t Disp = MI->getOperand(2).getImm(); 2730 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2731 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2732 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2733 DebugLoc DL = MI->getDebugLoc(); 2734 if (IsSubWord) 2735 BitSize = MI->getOperand(6).getImm(); 2736 2737 // Subword operations use 32-bit registers. 2738 const TargetRegisterClass *RC = (BitSize <= 32 ? 2739 &SystemZ::GR32BitRegClass : 2740 &SystemZ::GR64BitRegClass); 2741 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2742 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2743 2744 // Get the right opcodes for the displacement. 2745 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2746 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2747 assert(LOpcode && CSOpcode && "Displacement out of range"); 2748 2749 // Create virtual registers for temporary results. 2750 unsigned OrigVal = MRI.createVirtualRegister(RC); 2751 unsigned OldVal = MRI.createVirtualRegister(RC); 2752 unsigned NewVal = (BinOpcode || IsSubWord ? 2753 MRI.createVirtualRegister(RC) : Src2.getReg()); 2754 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2755 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2756 2757 // Insert a basic block for the main loop. 2758 MachineBasicBlock *StartMBB = MBB; 2759 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2760 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2761 2762 // StartMBB: 2763 // ... 2764 // %OrigVal = L Disp(%Base) 2765 // # fall through to LoopMMB 2766 MBB = StartMBB; 2767 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2768 .addOperand(Base).addImm(Disp).addReg(0); 2769 MBB->addSuccessor(LoopMBB); 2770 2771 // LoopMBB: 2772 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2773 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2774 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2775 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2776 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2777 // JNE LoopMBB 2778 // # fall through to DoneMMB 2779 MBB = LoopMBB; 2780 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2781 .addReg(OrigVal).addMBB(StartMBB) 2782 .addReg(Dest).addMBB(LoopMBB); 2783 if (IsSubWord) 2784 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2785 .addReg(OldVal).addReg(BitShift).addImm(0); 2786 if (Invert) { 2787 // Perform the operation normally and then invert every bit of the field. 2788 unsigned Tmp = MRI.createVirtualRegister(RC); 2789 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2790 .addReg(RotatedOldVal).addOperand(Src2); 2791 if (BitSize < 32) 2792 // XILF with the upper BitSize bits set. 2793 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2794 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 2795 else if (BitSize == 32) 2796 // XILF with every bit set. 2797 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2798 .addReg(Tmp).addImm(~uint32_t(0)); 2799 else { 2800 // Use LCGR and add -1 to the result, which is more compact than 2801 // an XILF, XILH pair. 2802 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2803 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2804 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2805 .addReg(Tmp2).addImm(-1); 2806 } 2807 } else if (BinOpcode) 2808 // A simply binary operation. 2809 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2810 .addReg(RotatedOldVal).addOperand(Src2); 2811 else if (IsSubWord) 2812 // Use RISBG to rotate Src2 into position and use it to replace the 2813 // field in RotatedOldVal. 2814 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2815 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2816 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2817 if (IsSubWord) 2818 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2819 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2820 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2821 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2822 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2823 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2824 MBB->addSuccessor(LoopMBB); 2825 MBB->addSuccessor(DoneMBB); 2826 2827 MI->eraseFromParent(); 2828 return DoneMBB; 2829} 2830 2831// Implement EmitInstrWithCustomInserter for pseudo 2832// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2833// instruction that should be used to compare the current field with the 2834// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2835// for when the current field should be kept. BitSize is the width of 2836// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2837MachineBasicBlock * 2838SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2839 MachineBasicBlock *MBB, 2840 unsigned CompareOpcode, 2841 unsigned KeepOldMask, 2842 unsigned BitSize) const { 2843 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2844 MachineFunction &MF = *MBB->getParent(); 2845 MachineRegisterInfo &MRI = MF.getRegInfo(); 2846 bool IsSubWord = (BitSize < 32); 2847 2848 // Extract the operands. Base can be a register or a frame index. 2849 unsigned Dest = MI->getOperand(0).getReg(); 2850 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2851 int64_t Disp = MI->getOperand(2).getImm(); 2852 unsigned Src2 = MI->getOperand(3).getReg(); 2853 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2854 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2855 DebugLoc DL = MI->getDebugLoc(); 2856 if (IsSubWord) 2857 BitSize = MI->getOperand(6).getImm(); 2858 2859 // Subword operations use 32-bit registers. 2860 const TargetRegisterClass *RC = (BitSize <= 32 ? 2861 &SystemZ::GR32BitRegClass : 2862 &SystemZ::GR64BitRegClass); 2863 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2864 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2865 2866 // Get the right opcodes for the displacement. 2867 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2868 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2869 assert(LOpcode && CSOpcode && "Displacement out of range"); 2870 2871 // Create virtual registers for temporary results. 2872 unsigned OrigVal = MRI.createVirtualRegister(RC); 2873 unsigned OldVal = MRI.createVirtualRegister(RC); 2874 unsigned NewVal = MRI.createVirtualRegister(RC); 2875 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2876 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2877 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2878 2879 // Insert 3 basic blocks for the loop. 2880 MachineBasicBlock *StartMBB = MBB; 2881 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2882 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2883 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2884 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2885 2886 // StartMBB: 2887 // ... 2888 // %OrigVal = L Disp(%Base) 2889 // # fall through to LoopMMB 2890 MBB = StartMBB; 2891 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2892 .addOperand(Base).addImm(Disp).addReg(0); 2893 MBB->addSuccessor(LoopMBB); 2894 2895 // LoopMBB: 2896 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2897 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2898 // CompareOpcode %RotatedOldVal, %Src2 2899 // BRC KeepOldMask, UpdateMBB 2900 MBB = LoopMBB; 2901 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2902 .addReg(OrigVal).addMBB(StartMBB) 2903 .addReg(Dest).addMBB(UpdateMBB); 2904 if (IsSubWord) 2905 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2906 .addReg(OldVal).addReg(BitShift).addImm(0); 2907 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2908 .addReg(RotatedOldVal).addReg(Src2); 2909 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2910 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2911 MBB->addSuccessor(UpdateMBB); 2912 MBB->addSuccessor(UseAltMBB); 2913 2914 // UseAltMBB: 2915 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2916 // # fall through to UpdateMMB 2917 MBB = UseAltMBB; 2918 if (IsSubWord) 2919 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2920 .addReg(RotatedOldVal).addReg(Src2) 2921 .addImm(32).addImm(31 + BitSize).addImm(0); 2922 MBB->addSuccessor(UpdateMBB); 2923 2924 // UpdateMBB: 2925 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2926 // [ %RotatedAltVal, UseAltMBB ] 2927 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2928 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2929 // JNE LoopMBB 2930 // # fall through to DoneMMB 2931 MBB = UpdateMBB; 2932 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2933 .addReg(RotatedOldVal).addMBB(LoopMBB) 2934 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2935 if (IsSubWord) 2936 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2937 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2938 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2939 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2940 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2941 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2942 MBB->addSuccessor(LoopMBB); 2943 MBB->addSuccessor(DoneMBB); 2944 2945 MI->eraseFromParent(); 2946 return DoneMBB; 2947} 2948 2949// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2950// instruction MI. 2951MachineBasicBlock * 2952SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2953 MachineBasicBlock *MBB) const { 2954 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2955 MachineFunction &MF = *MBB->getParent(); 2956 MachineRegisterInfo &MRI = MF.getRegInfo(); 2957 2958 // Extract the operands. Base can be a register or a frame index. 2959 unsigned Dest = MI->getOperand(0).getReg(); 2960 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2961 int64_t Disp = MI->getOperand(2).getImm(); 2962 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2963 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2964 unsigned BitShift = MI->getOperand(5).getReg(); 2965 unsigned NegBitShift = MI->getOperand(6).getReg(); 2966 int64_t BitSize = MI->getOperand(7).getImm(); 2967 DebugLoc DL = MI->getDebugLoc(); 2968 2969 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2970 2971 // Get the right opcodes for the displacement. 2972 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2973 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2974 assert(LOpcode && CSOpcode && "Displacement out of range"); 2975 2976 // Create virtual registers for temporary results. 2977 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2978 unsigned OldVal = MRI.createVirtualRegister(RC); 2979 unsigned CmpVal = MRI.createVirtualRegister(RC); 2980 unsigned SwapVal = MRI.createVirtualRegister(RC); 2981 unsigned StoreVal = MRI.createVirtualRegister(RC); 2982 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2983 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2984 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2985 2986 // Insert 2 basic blocks for the loop. 2987 MachineBasicBlock *StartMBB = MBB; 2988 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2989 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2990 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 2991 2992 // StartMBB: 2993 // ... 2994 // %OrigOldVal = L Disp(%Base) 2995 // # fall through to LoopMMB 2996 MBB = StartMBB; 2997 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 2998 .addOperand(Base).addImm(Disp).addReg(0); 2999 MBB->addSuccessor(LoopMBB); 3000 3001 // LoopMBB: 3002 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 3003 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 3004 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 3005 // %Dest = RLL %OldVal, BitSize(%BitShift) 3006 // ^^ The low BitSize bits contain the field 3007 // of interest. 3008 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 3009 // ^^ Replace the upper 32-BitSize bits of the 3010 // comparison value with those that we loaded, 3011 // so that we can use a full word comparison. 3012 // CR %Dest, %RetryCmpVal 3013 // JNE DoneMBB 3014 // # Fall through to SetMBB 3015 MBB = LoopMBB; 3016 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 3017 .addReg(OrigOldVal).addMBB(StartMBB) 3018 .addReg(RetryOldVal).addMBB(SetMBB); 3019 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 3020 .addReg(OrigCmpVal).addMBB(StartMBB) 3021 .addReg(RetryCmpVal).addMBB(SetMBB); 3022 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 3023 .addReg(OrigSwapVal).addMBB(StartMBB) 3024 .addReg(RetrySwapVal).addMBB(SetMBB); 3025 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 3026 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 3027 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 3028 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 3029 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 3030 .addReg(Dest).addReg(RetryCmpVal); 3031 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3032 .addImm(SystemZ::CCMASK_ICMP) 3033 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 3034 MBB->addSuccessor(DoneMBB); 3035 MBB->addSuccessor(SetMBB); 3036 3037 // SetMBB: 3038 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 3039 // ^^ Replace the upper 32-BitSize bits of the new 3040 // value with those that we loaded. 3041 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 3042 // ^^ Rotate the new field to its proper position. 3043 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 3044 // JNE LoopMBB 3045 // # fall through to ExitMMB 3046 MBB = SetMBB; 3047 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 3048 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 3049 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 3050 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 3051 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 3052 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 3053 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3054 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 3055 MBB->addSuccessor(LoopMBB); 3056 MBB->addSuccessor(DoneMBB); 3057 3058 MI->eraseFromParent(); 3059 return DoneMBB; 3060} 3061 3062// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 3063// if the high register of the GR128 value must be cleared or false if 3064// it's "don't care". SubReg is subreg_l32 when extending a GR32 3065// and subreg_l64 when extending a GR64. 3066MachineBasicBlock * 3067SystemZTargetLowering::emitExt128(MachineInstr *MI, 3068 MachineBasicBlock *MBB, 3069 bool ClearEven, unsigned SubReg) const { 3070 const SystemZInstrInfo *TII = TM.getInstrInfo(); 3071 MachineFunction &MF = *MBB->getParent(); 3072 MachineRegisterInfo &MRI = MF.getRegInfo(); 3073 DebugLoc DL = MI->getDebugLoc(); 3074 3075 unsigned Dest = MI->getOperand(0).getReg(); 3076 unsigned Src = MI->getOperand(1).getReg(); 3077 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 3078 3079 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 3080 if (ClearEven) { 3081 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 3082 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 3083 3084 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 3085 .addImm(0); 3086 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 3087 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64); 3088 In128 = NewIn128; 3089 } 3090 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 3091 .addReg(In128).addReg(Src).addImm(SubReg); 3092 3093 MI->eraseFromParent(); 3094 return MBB; 3095} 3096 3097MachineBasicBlock * 3098SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 3099 MachineBasicBlock *MBB, 3100 unsigned Opcode) const { 3101 const SystemZInstrInfo *TII = TM.getInstrInfo(); 3102 MachineFunction &MF = *MBB->getParent(); 3103 MachineRegisterInfo &MRI = MF.getRegInfo(); 3104 DebugLoc DL = MI->getDebugLoc(); 3105 3106 MachineOperand DestBase = earlyUseOperand(MI->getOperand(0)); 3107 uint64_t DestDisp = MI->getOperand(1).getImm(); 3108 MachineOperand SrcBase = earlyUseOperand(MI->getOperand(2)); 3109 uint64_t SrcDisp = MI->getOperand(3).getImm(); 3110 uint64_t Length = MI->getOperand(4).getImm(); 3111 3112 // When generating more than one CLC, all but the last will need to 3113 // branch to the end when a difference is found. 3114 MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ? 3115 splitBlockAfter(MI, MBB) : nullptr); 3116 3117 // Check for the loop form, in which operand 5 is the trip count. 3118 if (MI->getNumExplicitOperands() > 5) { 3119 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); 3120 3121 uint64_t StartCountReg = MI->getOperand(5).getReg(); 3122 uint64_t StartSrcReg = forceReg(MI, SrcBase, TII); 3123 uint64_t StartDestReg = (HaveSingleBase ? StartSrcReg : 3124 forceReg(MI, DestBase, TII)); 3125 3126 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass; 3127 uint64_t ThisSrcReg = MRI.createVirtualRegister(RC); 3128 uint64_t ThisDestReg = (HaveSingleBase ? ThisSrcReg : 3129 MRI.createVirtualRegister(RC)); 3130 uint64_t NextSrcReg = MRI.createVirtualRegister(RC); 3131 uint64_t NextDestReg = (HaveSingleBase ? NextSrcReg : 3132 MRI.createVirtualRegister(RC)); 3133 3134 RC = &SystemZ::GR64BitRegClass; 3135 uint64_t ThisCountReg = MRI.createVirtualRegister(RC); 3136 uint64_t NextCountReg = MRI.createVirtualRegister(RC); 3137 3138 MachineBasicBlock *StartMBB = MBB; 3139 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 3140 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 3141 MachineBasicBlock *NextMBB = (EndMBB ? emitBlockAfter(LoopMBB) : LoopMBB); 3142 3143 // StartMBB: 3144 // # fall through to LoopMMB 3145 MBB->addSuccessor(LoopMBB); 3146 3147 // LoopMBB: 3148 // %ThisDestReg = phi [ %StartDestReg, StartMBB ], 3149 // [ %NextDestReg, NextMBB ] 3150 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ], 3151 // [ %NextSrcReg, NextMBB ] 3152 // %ThisCountReg = phi [ %StartCountReg, StartMBB ], 3153 // [ %NextCountReg, NextMBB ] 3154 // ( PFD 2, 768+DestDisp(%ThisDestReg) ) 3155 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg) 3156 // ( JLH EndMBB ) 3157 // 3158 // The prefetch is used only for MVC. The JLH is used only for CLC. 3159 MBB = LoopMBB; 3160 3161 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) 3162 .addReg(StartDestReg).addMBB(StartMBB) 3163 .addReg(NextDestReg).addMBB(NextMBB); 3164 if (!HaveSingleBase) 3165 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) 3166 .addReg(StartSrcReg).addMBB(StartMBB) 3167 .addReg(NextSrcReg).addMBB(NextMBB); 3168 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) 3169 .addReg(StartCountReg).addMBB(StartMBB) 3170 .addReg(NextCountReg).addMBB(NextMBB); 3171 if (Opcode == SystemZ::MVC) 3172 BuildMI(MBB, DL, TII->get(SystemZ::PFD)) 3173 .addImm(SystemZ::PFD_WRITE) 3174 .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); 3175 BuildMI(MBB, DL, TII->get(Opcode)) 3176 .addReg(ThisDestReg).addImm(DestDisp).addImm(256) 3177 .addReg(ThisSrcReg).addImm(SrcDisp); 3178 if (EndMBB) { 3179 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3180 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3181 .addMBB(EndMBB); 3182 MBB->addSuccessor(EndMBB); 3183 MBB->addSuccessor(NextMBB); 3184 } 3185 3186 // NextMBB: 3187 // %NextDestReg = LA 256(%ThisDestReg) 3188 // %NextSrcReg = LA 256(%ThisSrcReg) 3189 // %NextCountReg = AGHI %ThisCountReg, -1 3190 // CGHI %NextCountReg, 0 3191 // JLH LoopMBB 3192 // # fall through to DoneMMB 3193 // 3194 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes. 3195 MBB = NextMBB; 3196 3197 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) 3198 .addReg(ThisDestReg).addImm(256).addReg(0); 3199 if (!HaveSingleBase) 3200 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) 3201 .addReg(ThisSrcReg).addImm(256).addReg(0); 3202 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) 3203 .addReg(ThisCountReg).addImm(-1); 3204 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) 3205 .addReg(NextCountReg).addImm(0); 3206 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3207 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3208 .addMBB(LoopMBB); 3209 MBB->addSuccessor(LoopMBB); 3210 MBB->addSuccessor(DoneMBB); 3211 3212 DestBase = MachineOperand::CreateReg(NextDestReg, false); 3213 SrcBase = MachineOperand::CreateReg(NextSrcReg, false); 3214 Length &= 255; 3215 MBB = DoneMBB; 3216 } 3217 // Handle any remaining bytes with straight-line code. 3218 while (Length > 0) { 3219 uint64_t ThisLength = std::min(Length, uint64_t(256)); 3220 // The previous iteration might have created out-of-range displacements. 3221 // Apply them using LAY if so. 3222 if (!isUInt<12>(DestDisp)) { 3223 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 3224 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 3225 .addOperand(DestBase).addImm(DestDisp).addReg(0); 3226 DestBase = MachineOperand::CreateReg(Reg, false); 3227 DestDisp = 0; 3228 } 3229 if (!isUInt<12>(SrcDisp)) { 3230 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 3231 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 3232 .addOperand(SrcBase).addImm(SrcDisp).addReg(0); 3233 SrcBase = MachineOperand::CreateReg(Reg, false); 3234 SrcDisp = 0; 3235 } 3236 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 3237 .addOperand(DestBase).addImm(DestDisp).addImm(ThisLength) 3238 .addOperand(SrcBase).addImm(SrcDisp); 3239 DestDisp += ThisLength; 3240 SrcDisp += ThisLength; 3241 Length -= ThisLength; 3242 // If there's another CLC to go, branch to the end if a difference 3243 // was found. 3244 if (EndMBB && Length > 0) { 3245 MachineBasicBlock *NextMBB = splitBlockBefore(MI, MBB); 3246 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3247 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3248 .addMBB(EndMBB); 3249 MBB->addSuccessor(EndMBB); 3250 MBB->addSuccessor(NextMBB); 3251 MBB = NextMBB; 3252 } 3253 } 3254 if (EndMBB) { 3255 MBB->addSuccessor(EndMBB); 3256 MBB = EndMBB; 3257 MBB->addLiveIn(SystemZ::CC); 3258 } 3259 3260 MI->eraseFromParent(); 3261 return MBB; 3262} 3263 3264// Decompose string pseudo-instruction MI into a loop that continually performs 3265// Opcode until CC != 3. 3266MachineBasicBlock * 3267SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 3268 MachineBasicBlock *MBB, 3269 unsigned Opcode) const { 3270 const SystemZInstrInfo *TII = TM.getInstrInfo(); 3271 MachineFunction &MF = *MBB->getParent(); 3272 MachineRegisterInfo &MRI = MF.getRegInfo(); 3273 DebugLoc DL = MI->getDebugLoc(); 3274 3275 uint64_t End1Reg = MI->getOperand(0).getReg(); 3276 uint64_t Start1Reg = MI->getOperand(1).getReg(); 3277 uint64_t Start2Reg = MI->getOperand(2).getReg(); 3278 uint64_t CharReg = MI->getOperand(3).getReg(); 3279 3280 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 3281 uint64_t This1Reg = MRI.createVirtualRegister(RC); 3282 uint64_t This2Reg = MRI.createVirtualRegister(RC); 3283 uint64_t End2Reg = MRI.createVirtualRegister(RC); 3284 3285 MachineBasicBlock *StartMBB = MBB; 3286 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 3287 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 3288 3289 // StartMBB: 3290 // # fall through to LoopMMB 3291 MBB->addSuccessor(LoopMBB); 3292 3293 // LoopMBB: 3294 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 3295 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 3296 // R0L = %CharReg 3297 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L 3298 // JO LoopMBB 3299 // # fall through to DoneMMB 3300 // 3301 // The load of R0L can be hoisted by post-RA LICM. 3302 MBB = LoopMBB; 3303 3304 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 3305 .addReg(Start1Reg).addMBB(StartMBB) 3306 .addReg(End1Reg).addMBB(LoopMBB); 3307 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 3308 .addReg(Start2Reg).addMBB(StartMBB) 3309 .addReg(End2Reg).addMBB(LoopMBB); 3310 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); 3311 BuildMI(MBB, DL, TII->get(Opcode)) 3312 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 3313 .addReg(This1Reg).addReg(This2Reg); 3314 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3315 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 3316 MBB->addSuccessor(LoopMBB); 3317 MBB->addSuccessor(DoneMBB); 3318 3319 DoneMBB->addLiveIn(SystemZ::CC); 3320 3321 MI->eraseFromParent(); 3322 return DoneMBB; 3323} 3324 3325MachineBasicBlock *SystemZTargetLowering:: 3326EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 3327 switch (MI->getOpcode()) { 3328 case SystemZ::Select32Mux: 3329 case SystemZ::Select32: 3330 case SystemZ::SelectF32: 3331 case SystemZ::Select64: 3332 case SystemZ::SelectF64: 3333 case SystemZ::SelectF128: 3334 return emitSelect(MI, MBB); 3335 3336 case SystemZ::CondStore8Mux: 3337 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false); 3338 case SystemZ::CondStore8MuxInv: 3339 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true); 3340 case SystemZ::CondStore16Mux: 3341 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false); 3342 case SystemZ::CondStore16MuxInv: 3343 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true); 3344 case SystemZ::CondStore8: 3345 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 3346 case SystemZ::CondStore8Inv: 3347 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 3348 case SystemZ::CondStore16: 3349 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 3350 case SystemZ::CondStore16Inv: 3351 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 3352 case SystemZ::CondStore32: 3353 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 3354 case SystemZ::CondStore32Inv: 3355 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 3356 case SystemZ::CondStore64: 3357 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 3358 case SystemZ::CondStore64Inv: 3359 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 3360 case SystemZ::CondStoreF32: 3361 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 3362 case SystemZ::CondStoreF32Inv: 3363 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 3364 case SystemZ::CondStoreF64: 3365 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 3366 case SystemZ::CondStoreF64Inv: 3367 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 3368 3369 case SystemZ::AEXT128_64: 3370 return emitExt128(MI, MBB, false, SystemZ::subreg_l64); 3371 case SystemZ::ZEXT128_32: 3372 return emitExt128(MI, MBB, true, SystemZ::subreg_l32); 3373 case SystemZ::ZEXT128_64: 3374 return emitExt128(MI, MBB, true, SystemZ::subreg_l64); 3375 3376 case SystemZ::ATOMIC_SWAPW: 3377 return emitAtomicLoadBinary(MI, MBB, 0, 0); 3378 case SystemZ::ATOMIC_SWAP_32: 3379 return emitAtomicLoadBinary(MI, MBB, 0, 32); 3380 case SystemZ::ATOMIC_SWAP_64: 3381 return emitAtomicLoadBinary(MI, MBB, 0, 64); 3382 3383 case SystemZ::ATOMIC_LOADW_AR: 3384 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 3385 case SystemZ::ATOMIC_LOADW_AFI: 3386 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 3387 case SystemZ::ATOMIC_LOAD_AR: 3388 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 3389 case SystemZ::ATOMIC_LOAD_AHI: 3390 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 3391 case SystemZ::ATOMIC_LOAD_AFI: 3392 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 3393 case SystemZ::ATOMIC_LOAD_AGR: 3394 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 3395 case SystemZ::ATOMIC_LOAD_AGHI: 3396 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 3397 case SystemZ::ATOMIC_LOAD_AGFI: 3398 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 3399 3400 case SystemZ::ATOMIC_LOADW_SR: 3401 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 3402 case SystemZ::ATOMIC_LOAD_SR: 3403 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 3404 case SystemZ::ATOMIC_LOAD_SGR: 3405 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 3406 3407 case SystemZ::ATOMIC_LOADW_NR: 3408 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 3409 case SystemZ::ATOMIC_LOADW_NILH: 3410 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); 3411 case SystemZ::ATOMIC_LOAD_NR: 3412 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 3413 case SystemZ::ATOMIC_LOAD_NILL: 3414 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); 3415 case SystemZ::ATOMIC_LOAD_NILH: 3416 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); 3417 case SystemZ::ATOMIC_LOAD_NILF: 3418 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); 3419 case SystemZ::ATOMIC_LOAD_NGR: 3420 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 3421 case SystemZ::ATOMIC_LOAD_NILL64: 3422 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); 3423 case SystemZ::ATOMIC_LOAD_NILH64: 3424 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); 3425 case SystemZ::ATOMIC_LOAD_NIHL64: 3426 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); 3427 case SystemZ::ATOMIC_LOAD_NIHH64: 3428 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); 3429 case SystemZ::ATOMIC_LOAD_NILF64: 3430 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); 3431 case SystemZ::ATOMIC_LOAD_NIHF64: 3432 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); 3433 3434 case SystemZ::ATOMIC_LOADW_OR: 3435 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 3436 case SystemZ::ATOMIC_LOADW_OILH: 3437 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); 3438 case SystemZ::ATOMIC_LOAD_OR: 3439 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 3440 case SystemZ::ATOMIC_LOAD_OILL: 3441 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); 3442 case SystemZ::ATOMIC_LOAD_OILH: 3443 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); 3444 case SystemZ::ATOMIC_LOAD_OILF: 3445 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); 3446 case SystemZ::ATOMIC_LOAD_OGR: 3447 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 3448 case SystemZ::ATOMIC_LOAD_OILL64: 3449 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); 3450 case SystemZ::ATOMIC_LOAD_OILH64: 3451 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); 3452 case SystemZ::ATOMIC_LOAD_OIHL64: 3453 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); 3454 case SystemZ::ATOMIC_LOAD_OIHH64: 3455 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); 3456 case SystemZ::ATOMIC_LOAD_OILF64: 3457 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); 3458 case SystemZ::ATOMIC_LOAD_OIHF64: 3459 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); 3460 3461 case SystemZ::ATOMIC_LOADW_XR: 3462 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 3463 case SystemZ::ATOMIC_LOADW_XILF: 3464 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); 3465 case SystemZ::ATOMIC_LOAD_XR: 3466 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 3467 case SystemZ::ATOMIC_LOAD_XILF: 3468 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); 3469 case SystemZ::ATOMIC_LOAD_XGR: 3470 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 3471 case SystemZ::ATOMIC_LOAD_XILF64: 3472 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); 3473 case SystemZ::ATOMIC_LOAD_XIHF64: 3474 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); 3475 3476 case SystemZ::ATOMIC_LOADW_NRi: 3477 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 3478 case SystemZ::ATOMIC_LOADW_NILHi: 3479 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); 3480 case SystemZ::ATOMIC_LOAD_NRi: 3481 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 3482 case SystemZ::ATOMIC_LOAD_NILLi: 3483 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); 3484 case SystemZ::ATOMIC_LOAD_NILHi: 3485 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); 3486 case SystemZ::ATOMIC_LOAD_NILFi: 3487 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); 3488 case SystemZ::ATOMIC_LOAD_NGRi: 3489 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 3490 case SystemZ::ATOMIC_LOAD_NILL64i: 3491 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); 3492 case SystemZ::ATOMIC_LOAD_NILH64i: 3493 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); 3494 case SystemZ::ATOMIC_LOAD_NIHL64i: 3495 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); 3496 case SystemZ::ATOMIC_LOAD_NIHH64i: 3497 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); 3498 case SystemZ::ATOMIC_LOAD_NILF64i: 3499 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); 3500 case SystemZ::ATOMIC_LOAD_NIHF64i: 3501 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); 3502 3503 case SystemZ::ATOMIC_LOADW_MIN: 3504 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3505 SystemZ::CCMASK_CMP_LE, 0); 3506 case SystemZ::ATOMIC_LOAD_MIN_32: 3507 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3508 SystemZ::CCMASK_CMP_LE, 32); 3509 case SystemZ::ATOMIC_LOAD_MIN_64: 3510 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3511 SystemZ::CCMASK_CMP_LE, 64); 3512 3513 case SystemZ::ATOMIC_LOADW_MAX: 3514 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3515 SystemZ::CCMASK_CMP_GE, 0); 3516 case SystemZ::ATOMIC_LOAD_MAX_32: 3517 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3518 SystemZ::CCMASK_CMP_GE, 32); 3519 case SystemZ::ATOMIC_LOAD_MAX_64: 3520 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3521 SystemZ::CCMASK_CMP_GE, 64); 3522 3523 case SystemZ::ATOMIC_LOADW_UMIN: 3524 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3525 SystemZ::CCMASK_CMP_LE, 0); 3526 case SystemZ::ATOMIC_LOAD_UMIN_32: 3527 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3528 SystemZ::CCMASK_CMP_LE, 32); 3529 case SystemZ::ATOMIC_LOAD_UMIN_64: 3530 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3531 SystemZ::CCMASK_CMP_LE, 64); 3532 3533 case SystemZ::ATOMIC_LOADW_UMAX: 3534 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3535 SystemZ::CCMASK_CMP_GE, 0); 3536 case SystemZ::ATOMIC_LOAD_UMAX_32: 3537 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3538 SystemZ::CCMASK_CMP_GE, 32); 3539 case SystemZ::ATOMIC_LOAD_UMAX_64: 3540 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3541 SystemZ::CCMASK_CMP_GE, 64); 3542 3543 case SystemZ::ATOMIC_CMP_SWAPW: 3544 return emitAtomicCmpSwapW(MI, MBB); 3545 case SystemZ::MVCSequence: 3546 case SystemZ::MVCLoop: 3547 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 3548 case SystemZ::NCSequence: 3549 case SystemZ::NCLoop: 3550 return emitMemMemWrapper(MI, MBB, SystemZ::NC); 3551 case SystemZ::OCSequence: 3552 case SystemZ::OCLoop: 3553 return emitMemMemWrapper(MI, MBB, SystemZ::OC); 3554 case SystemZ::XCSequence: 3555 case SystemZ::XCLoop: 3556 return emitMemMemWrapper(MI, MBB, SystemZ::XC); 3557 case SystemZ::CLCSequence: 3558 case SystemZ::CLCLoop: 3559 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 3560 case SystemZ::CLSTLoop: 3561 return emitStringWrapper(MI, MBB, SystemZ::CLST); 3562 case SystemZ::MVSTLoop: 3563 return emitStringWrapper(MI, MBB, SystemZ::MVST); 3564 case SystemZ::SRSTLoop: 3565 return emitStringWrapper(MI, MBB, SystemZ::SRST); 3566 default: 3567 llvm_unreachable("Unexpected instr type to insert"); 3568 } 3569} 3570