SystemZISelLowering.cpp revision 1c831f7f1f7e869516f5a70c1e6dd8896bef76f5
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "systemz-lower" 15 16#include "SystemZISelLowering.h" 17#include "SystemZCallingConv.h" 18#include "SystemZConstantPoolValue.h" 19#include "SystemZMachineFunctionInfo.h" 20#include "SystemZTargetMachine.h" 21#include "llvm/CodeGen/CallingConvLower.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25 26using namespace llvm; 27 28// Classify VT as either 32 or 64 bit. 29static bool is32Bit(EVT VT) { 30 switch (VT.getSimpleVT().SimpleTy) { 31 case MVT::i32: 32 return true; 33 case MVT::i64: 34 return false; 35 default: 36 llvm_unreachable("Unsupported type"); 37 } 38} 39 40// Return a version of MachineOperand that can be safely used before the 41// final use. 42static MachineOperand earlyUseOperand(MachineOperand Op) { 43 if (Op.isReg()) 44 Op.setIsKill(false); 45 return Op; 46} 47 48SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 49 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 50 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 51 MVT PtrVT = getPointerTy(); 52 53 // Set up the register classes. 54 if (Subtarget.hasHighWord()) 55 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 56 else 57 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 58 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 59 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 60 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 61 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 62 63 // Compute derived properties from the register classes 64 computeRegisterProperties(); 65 66 // Set up special registers. 67 setExceptionPointerRegister(SystemZ::R6D); 68 setExceptionSelectorRegister(SystemZ::R7D); 69 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 70 71 // TODO: It may be better to default to latency-oriented scheduling, however 72 // LLVM's current latency-oriented scheduler can't handle physreg definitions 73 // such as SystemZ has with CC, so set this to the register-pressure 74 // scheduler, because it can. 75 setSchedulingPreference(Sched::RegPressure); 76 77 setBooleanContents(ZeroOrOneBooleanContent); 78 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 79 80 // Instructions are strings of 2-byte aligned 2-byte values. 81 setMinFunctionAlignment(2); 82 83 // Handle operations that are handled in a similar way for all types. 84 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 85 I <= MVT::LAST_FP_VALUETYPE; 86 ++I) { 87 MVT VT = MVT::SimpleValueType(I); 88 if (isTypeLegal(VT)) { 89 // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). 90 setOperationAction(ISD::SETCC, VT, Expand); 91 92 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 93 setOperationAction(ISD::SELECT, VT, Expand); 94 95 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 96 setOperationAction(ISD::SELECT_CC, VT, Custom); 97 setOperationAction(ISD::BR_CC, VT, Custom); 98 } 99 } 100 101 // Expand jump table branches as address arithmetic followed by an 102 // indirect jump. 103 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 104 105 // Expand BRCOND into a BR_CC (see above). 106 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 107 108 // Handle integer types. 109 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 110 I <= MVT::LAST_INTEGER_VALUETYPE; 111 ++I) { 112 MVT VT = MVT::SimpleValueType(I); 113 if (isTypeLegal(VT)) { 114 // Expand individual DIV and REMs into DIVREMs. 115 setOperationAction(ISD::SDIV, VT, Expand); 116 setOperationAction(ISD::UDIV, VT, Expand); 117 setOperationAction(ISD::SREM, VT, Expand); 118 setOperationAction(ISD::UREM, VT, Expand); 119 setOperationAction(ISD::SDIVREM, VT, Custom); 120 setOperationAction(ISD::UDIVREM, VT, Custom); 121 122 // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 123 // FIXME: probably much too conservative. 124 setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 125 setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 126 127 // No special instructions for these. 128 setOperationAction(ISD::CTPOP, VT, Expand); 129 setOperationAction(ISD::CTTZ, VT, Expand); 130 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 131 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 132 setOperationAction(ISD::ROTR, VT, Expand); 133 134 // Use *MUL_LOHI where possible instead of MULH*. 135 setOperationAction(ISD::MULHS, VT, Expand); 136 setOperationAction(ISD::MULHU, VT, Expand); 137 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 138 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 139 140 // We have instructions for signed but not unsigned FP conversion. 141 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 142 } 143 } 144 145 // Type legalization will convert 8- and 16-bit atomic operations into 146 // forms that operate on i32s (but still keeping the original memory VT). 147 // Lower them into full i32 operations. 148 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 149 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 150 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 151 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 152 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 153 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 154 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 155 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 156 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 157 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 158 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 159 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 160 161 // We have instructions for signed but not unsigned FP conversion. 162 // Handle unsigned 32-bit types as signed 64-bit types. 163 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 164 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 165 166 // We have native support for a 64-bit CTLZ, via FLOGR. 167 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 168 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 169 170 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 171 setOperationAction(ISD::OR, MVT::i64, Custom); 172 173 // FIXME: Can we support these natively? 174 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 175 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 176 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 177 178 // We have native instructions for i8, i16 and i32 extensions, but not i1. 179 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 180 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 181 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 182 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 183 184 // Handle the various types of symbolic address. 185 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 186 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 187 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 188 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 189 setOperationAction(ISD::JumpTable, PtrVT, Custom); 190 191 // We need to handle dynamic allocations specially because of the 192 // 160-byte area at the bottom of the stack. 193 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 194 195 // Use custom expanders so that we can force the function to use 196 // a frame pointer. 197 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 198 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 199 200 // Handle prefetches with PFD or PFDRL. 201 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 202 203 // Handle floating-point types. 204 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 205 I <= MVT::LAST_FP_VALUETYPE; 206 ++I) { 207 MVT VT = MVT::SimpleValueType(I); 208 if (isTypeLegal(VT)) { 209 // We can use FI for FRINT. 210 setOperationAction(ISD::FRINT, VT, Legal); 211 212 // We can use the extended form of FI for other rounding operations. 213 if (Subtarget.hasFPExtension()) { 214 setOperationAction(ISD::FNEARBYINT, VT, Legal); 215 setOperationAction(ISD::FFLOOR, VT, Legal); 216 setOperationAction(ISD::FCEIL, VT, Legal); 217 setOperationAction(ISD::FTRUNC, VT, Legal); 218 setOperationAction(ISD::FROUND, VT, Legal); 219 } 220 221 // No special instructions for these. 222 setOperationAction(ISD::FSIN, VT, Expand); 223 setOperationAction(ISD::FCOS, VT, Expand); 224 setOperationAction(ISD::FREM, VT, Expand); 225 } 226 } 227 228 // We have fused multiply-addition for f32 and f64 but not f128. 229 setOperationAction(ISD::FMA, MVT::f32, Legal); 230 setOperationAction(ISD::FMA, MVT::f64, Legal); 231 setOperationAction(ISD::FMA, MVT::f128, Expand); 232 233 // Needed so that we don't try to implement f128 constant loads using 234 // a load-and-extend of a f80 constant (in cases where the constant 235 // would fit in an f80). 236 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 237 238 // Floating-point truncation and stores need to be done separately. 239 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 240 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 241 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 242 243 // We have 64-bit FPR<->GPR moves, but need special handling for 244 // 32-bit forms. 245 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 246 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 247 248 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 249 // structure, but VAEND is a no-op. 250 setOperationAction(ISD::VASTART, MVT::Other, Custom); 251 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 252 setOperationAction(ISD::VAEND, MVT::Other, Expand); 253 254 // We want to use MVC in preference to even a single load/store pair. 255 MaxStoresPerMemcpy = 0; 256 MaxStoresPerMemcpyOptSize = 0; 257 258 // The main memset sequence is a byte store followed by an MVC. 259 // Two STC or MV..I stores win over that, but the kind of fused stores 260 // generated by target-independent code don't when the byte value is 261 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 262 // than "STC;MVC". Handle the choice in target-specific code instead. 263 MaxStoresPerMemset = 0; 264 MaxStoresPerMemsetOptSize = 0; 265} 266 267bool 268SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 269 VT = VT.getScalarType(); 270 271 if (!VT.isSimple()) 272 return false; 273 274 switch (VT.getSimpleVT().SimpleTy) { 275 case MVT::f32: 276 case MVT::f64: 277 return true; 278 case MVT::f128: 279 return false; 280 default: 281 break; 282 } 283 284 return false; 285} 286 287bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 288 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 289 return Imm.isZero() || Imm.isNegZero(); 290} 291 292bool SystemZTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 293 bool *Fast) const { 294 // Unaligned accesses should never be slower than the expanded version. 295 // We check specifically for aligned accesses in the few cases where 296 // they are required. 297 if (Fast) 298 *Fast = true; 299 return true; 300} 301 302bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 303 Type *Ty) const { 304 // Punt on globals for now, although they can be used in limited 305 // RELATIVE LONG cases. 306 if (AM.BaseGV) 307 return false; 308 309 // Require a 20-bit signed offset. 310 if (!isInt<20>(AM.BaseOffs)) 311 return false; 312 313 // Indexing is OK but no scale factor can be applied. 314 return AM.Scale == 0 || AM.Scale == 1; 315} 316 317bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 318 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 319 return false; 320 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 321 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 322 return FromBits > ToBits; 323} 324 325bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 326 if (!FromVT.isInteger() || !ToVT.isInteger()) 327 return false; 328 unsigned FromBits = FromVT.getSizeInBits(); 329 unsigned ToBits = ToVT.getSizeInBits(); 330 return FromBits > ToBits; 331} 332 333//===----------------------------------------------------------------------===// 334// Inline asm support 335//===----------------------------------------------------------------------===// 336 337TargetLowering::ConstraintType 338SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 339 if (Constraint.size() == 1) { 340 switch (Constraint[0]) { 341 case 'a': // Address register 342 case 'd': // Data register (equivalent to 'r') 343 case 'f': // Floating-point register 344 case 'h': // High-part register 345 case 'r': // General-purpose register 346 return C_RegisterClass; 347 348 case 'Q': // Memory with base and unsigned 12-bit displacement 349 case 'R': // Likewise, plus an index 350 case 'S': // Memory with base and signed 20-bit displacement 351 case 'T': // Likewise, plus an index 352 case 'm': // Equivalent to 'T'. 353 return C_Memory; 354 355 case 'I': // Unsigned 8-bit constant 356 case 'J': // Unsigned 12-bit constant 357 case 'K': // Signed 16-bit constant 358 case 'L': // Signed 20-bit displacement (on all targets we support) 359 case 'M': // 0x7fffffff 360 return C_Other; 361 362 default: 363 break; 364 } 365 } 366 return TargetLowering::getConstraintType(Constraint); 367} 368 369TargetLowering::ConstraintWeight SystemZTargetLowering:: 370getSingleConstraintMatchWeight(AsmOperandInfo &info, 371 const char *constraint) const { 372 ConstraintWeight weight = CW_Invalid; 373 Value *CallOperandVal = info.CallOperandVal; 374 // If we don't have a value, we can't do a match, 375 // but allow it at the lowest weight. 376 if (CallOperandVal == NULL) 377 return CW_Default; 378 Type *type = CallOperandVal->getType(); 379 // Look at the constraint type. 380 switch (*constraint) { 381 default: 382 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 383 break; 384 385 case 'a': // Address register 386 case 'd': // Data register (equivalent to 'r') 387 case 'h': // High-part register 388 case 'r': // General-purpose register 389 if (CallOperandVal->getType()->isIntegerTy()) 390 weight = CW_Register; 391 break; 392 393 case 'f': // Floating-point register 394 if (type->isFloatingPointTy()) 395 weight = CW_Register; 396 break; 397 398 case 'I': // Unsigned 8-bit constant 399 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 400 if (isUInt<8>(C->getZExtValue())) 401 weight = CW_Constant; 402 break; 403 404 case 'J': // Unsigned 12-bit constant 405 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 406 if (isUInt<12>(C->getZExtValue())) 407 weight = CW_Constant; 408 break; 409 410 case 'K': // Signed 16-bit constant 411 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 412 if (isInt<16>(C->getSExtValue())) 413 weight = CW_Constant; 414 break; 415 416 case 'L': // Signed 20-bit displacement (on all targets we support) 417 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 418 if (isInt<20>(C->getSExtValue())) 419 weight = CW_Constant; 420 break; 421 422 case 'M': // 0x7fffffff 423 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 424 if (C->getZExtValue() == 0x7fffffff) 425 weight = CW_Constant; 426 break; 427 } 428 return weight; 429} 430 431// Parse a "{tNNN}" register constraint for which the register type "t" 432// has already been verified. MC is the class associated with "t" and 433// Map maps 0-based register numbers to LLVM register numbers. 434static std::pair<unsigned, const TargetRegisterClass *> 435parseRegisterNumber(const std::string &Constraint, 436 const TargetRegisterClass *RC, const unsigned *Map) { 437 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 438 if (isdigit(Constraint[2])) { 439 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 440 unsigned Index = atoi(Suffix.c_str()); 441 if (Index < 16 && Map[Index]) 442 return std::make_pair(Map[Index], RC); 443 } 444 return std::make_pair(0u, static_cast<TargetRegisterClass*>(0)); 445} 446 447std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 448getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 449 if (Constraint.size() == 1) { 450 // GCC Constraint Letters 451 switch (Constraint[0]) { 452 default: break; 453 case 'd': // Data register (equivalent to 'r') 454 case 'r': // General-purpose register 455 if (VT == MVT::i64) 456 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 457 else if (VT == MVT::i128) 458 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 459 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 460 461 case 'a': // Address register 462 if (VT == MVT::i64) 463 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 464 else if (VT == MVT::i128) 465 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 466 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 467 468 case 'h': // High-part register (an LLVM extension) 469 return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 470 471 case 'f': // Floating-point register 472 if (VT == MVT::f64) 473 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 474 else if (VT == MVT::f128) 475 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 476 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 477 } 478 } 479 if (Constraint[0] == '{') { 480 // We need to override the default register parsing for GPRs and FPRs 481 // because the interpretation depends on VT. The internal names of 482 // the registers are also different from the external names 483 // (F0D and F0S instead of F0, etc.). 484 if (Constraint[1] == 'r') { 485 if (VT == MVT::i32) 486 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 487 SystemZMC::GR32Regs); 488 if (VT == MVT::i128) 489 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 490 SystemZMC::GR128Regs); 491 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 492 SystemZMC::GR64Regs); 493 } 494 if (Constraint[1] == 'f') { 495 if (VT == MVT::f32) 496 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 497 SystemZMC::FP32Regs); 498 if (VT == MVT::f128) 499 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 500 SystemZMC::FP128Regs); 501 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 502 SystemZMC::FP64Regs); 503 } 504 } 505 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 506} 507 508void SystemZTargetLowering:: 509LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 510 std::vector<SDValue> &Ops, 511 SelectionDAG &DAG) const { 512 // Only support length 1 constraints for now. 513 if (Constraint.length() == 1) { 514 switch (Constraint[0]) { 515 case 'I': // Unsigned 8-bit constant 516 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 517 if (isUInt<8>(C->getZExtValue())) 518 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 519 Op.getValueType())); 520 return; 521 522 case 'J': // Unsigned 12-bit constant 523 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 524 if (isUInt<12>(C->getZExtValue())) 525 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 526 Op.getValueType())); 527 return; 528 529 case 'K': // Signed 16-bit constant 530 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 531 if (isInt<16>(C->getSExtValue())) 532 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 533 Op.getValueType())); 534 return; 535 536 case 'L': // Signed 20-bit displacement (on all targets we support) 537 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 538 if (isInt<20>(C->getSExtValue())) 539 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 540 Op.getValueType())); 541 return; 542 543 case 'M': // 0x7fffffff 544 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 545 if (C->getZExtValue() == 0x7fffffff) 546 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 547 Op.getValueType())); 548 return; 549 } 550 } 551 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 552} 553 554//===----------------------------------------------------------------------===// 555// Calling conventions 556//===----------------------------------------------------------------------===// 557 558#include "SystemZGenCallingConv.inc" 559 560bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 561 Type *ToType) const { 562 return isTruncateFree(FromType, ToType); 563} 564 565bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 566 if (!CI->isTailCall()) 567 return false; 568 return true; 569} 570 571// Value is a value that has been passed to us in the location described by VA 572// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 573// any loads onto Chain. 574static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 575 CCValAssign &VA, SDValue Chain, 576 SDValue Value) { 577 // If the argument has been promoted from a smaller type, insert an 578 // assertion to capture this. 579 if (VA.getLocInfo() == CCValAssign::SExt) 580 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 581 DAG.getValueType(VA.getValVT())); 582 else if (VA.getLocInfo() == CCValAssign::ZExt) 583 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 584 DAG.getValueType(VA.getValVT())); 585 586 if (VA.isExtInLoc()) 587 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 588 else if (VA.getLocInfo() == CCValAssign::Indirect) 589 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 590 MachinePointerInfo(), false, false, false, 0); 591 else 592 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 593 return Value; 594} 595 596// Value is a value of type VA.getValVT() that we need to copy into 597// the location described by VA. Return a copy of Value converted to 598// VA.getValVT(). The caller is responsible for handling indirect values. 599static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 600 CCValAssign &VA, SDValue Value) { 601 switch (VA.getLocInfo()) { 602 case CCValAssign::SExt: 603 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 604 case CCValAssign::ZExt: 605 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 606 case CCValAssign::AExt: 607 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 608 case CCValAssign::Full: 609 return Value; 610 default: 611 llvm_unreachable("Unhandled getLocInfo()"); 612 } 613} 614 615SDValue SystemZTargetLowering:: 616LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 617 const SmallVectorImpl<ISD::InputArg> &Ins, 618 SDLoc DL, SelectionDAG &DAG, 619 SmallVectorImpl<SDValue> &InVals) const { 620 MachineFunction &MF = DAG.getMachineFunction(); 621 MachineFrameInfo *MFI = MF.getFrameInfo(); 622 MachineRegisterInfo &MRI = MF.getRegInfo(); 623 SystemZMachineFunctionInfo *FuncInfo = 624 MF.getInfo<SystemZMachineFunctionInfo>(); 625 const SystemZFrameLowering *TFL = 626 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 627 628 // Assign locations to all of the incoming arguments. 629 SmallVector<CCValAssign, 16> ArgLocs; 630 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 631 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 632 633 unsigned NumFixedGPRs = 0; 634 unsigned NumFixedFPRs = 0; 635 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 636 SDValue ArgValue; 637 CCValAssign &VA = ArgLocs[I]; 638 EVT LocVT = VA.getLocVT(); 639 if (VA.isRegLoc()) { 640 // Arguments passed in registers 641 const TargetRegisterClass *RC; 642 switch (LocVT.getSimpleVT().SimpleTy) { 643 default: 644 // Integers smaller than i64 should be promoted to i64. 645 llvm_unreachable("Unexpected argument type"); 646 case MVT::i32: 647 NumFixedGPRs += 1; 648 RC = &SystemZ::GR32BitRegClass; 649 break; 650 case MVT::i64: 651 NumFixedGPRs += 1; 652 RC = &SystemZ::GR64BitRegClass; 653 break; 654 case MVT::f32: 655 NumFixedFPRs += 1; 656 RC = &SystemZ::FP32BitRegClass; 657 break; 658 case MVT::f64: 659 NumFixedFPRs += 1; 660 RC = &SystemZ::FP64BitRegClass; 661 break; 662 } 663 664 unsigned VReg = MRI.createVirtualRegister(RC); 665 MRI.addLiveIn(VA.getLocReg(), VReg); 666 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 667 } else { 668 assert(VA.isMemLoc() && "Argument not register or memory"); 669 670 // Create the frame index object for this incoming parameter. 671 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 672 VA.getLocMemOffset(), true); 673 674 // Create the SelectionDAG nodes corresponding to a load 675 // from this parameter. Unpromoted ints and floats are 676 // passed as right-justified 8-byte values. 677 EVT PtrVT = getPointerTy(); 678 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 679 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 680 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 681 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 682 MachinePointerInfo::getFixedStack(FI), 683 false, false, false, 0); 684 } 685 686 // Convert the value of the argument register into the value that's 687 // being passed. 688 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 689 } 690 691 if (IsVarArg) { 692 // Save the number of non-varargs registers for later use by va_start, etc. 693 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 694 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 695 696 // Likewise the address (in the form of a frame index) of where the 697 // first stack vararg would be. The 1-byte size here is arbitrary. 698 int64_t StackSize = CCInfo.getNextStackOffset(); 699 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 700 701 // ...and a similar frame index for the caller-allocated save area 702 // that will be used to store the incoming registers. 703 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 704 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 705 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 706 707 // Store the FPR varargs in the reserved frame slots. (We store the 708 // GPRs as part of the prologue.) 709 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 710 SDValue MemOps[SystemZ::NumArgFPRs]; 711 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 712 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 713 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 714 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 715 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 716 &SystemZ::FP64BitRegClass); 717 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 718 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 719 MachinePointerInfo::getFixedStack(FI), 720 false, false, 0); 721 722 } 723 // Join the stores, which are independent of one another. 724 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 725 &MemOps[NumFixedFPRs], 726 SystemZ::NumArgFPRs - NumFixedFPRs); 727 } 728 } 729 730 return Chain; 731} 732 733static bool canUseSiblingCall(CCState ArgCCInfo, 734 SmallVectorImpl<CCValAssign> &ArgLocs) { 735 // Punt if there are any indirect or stack arguments, or if the call 736 // needs the call-saved argument register R6. 737 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 738 CCValAssign &VA = ArgLocs[I]; 739 if (VA.getLocInfo() == CCValAssign::Indirect) 740 return false; 741 if (!VA.isRegLoc()) 742 return false; 743 unsigned Reg = VA.getLocReg(); 744 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 745 return false; 746 } 747 return true; 748} 749 750SDValue 751SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 752 SmallVectorImpl<SDValue> &InVals) const { 753 SelectionDAG &DAG = CLI.DAG; 754 SDLoc &DL = CLI.DL; 755 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 756 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 757 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 758 SDValue Chain = CLI.Chain; 759 SDValue Callee = CLI.Callee; 760 bool &IsTailCall = CLI.IsTailCall; 761 CallingConv::ID CallConv = CLI.CallConv; 762 bool IsVarArg = CLI.IsVarArg; 763 MachineFunction &MF = DAG.getMachineFunction(); 764 EVT PtrVT = getPointerTy(); 765 766 // Analyze the operands of the call, assigning locations to each operand. 767 SmallVector<CCValAssign, 16> ArgLocs; 768 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 769 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 770 771 // We don't support GuaranteedTailCallOpt, only automatically-detected 772 // sibling calls. 773 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 774 IsTailCall = false; 775 776 // Get a count of how many bytes are to be pushed on the stack. 777 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 778 779 // Mark the start of the call. 780 if (!IsTailCall) 781 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 782 DL); 783 784 // Copy argument values to their designated locations. 785 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 786 SmallVector<SDValue, 8> MemOpChains; 787 SDValue StackPtr; 788 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 789 CCValAssign &VA = ArgLocs[I]; 790 SDValue ArgValue = OutVals[I]; 791 792 if (VA.getLocInfo() == CCValAssign::Indirect) { 793 // Store the argument in a stack slot and pass its address. 794 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 795 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 796 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 797 MachinePointerInfo::getFixedStack(FI), 798 false, false, 0)); 799 ArgValue = SpillSlot; 800 } else 801 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 802 803 if (VA.isRegLoc()) 804 // Queue up the argument copies and emit them at the end. 805 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 806 else { 807 assert(VA.isMemLoc() && "Argument not register or memory"); 808 809 // Work out the address of the stack slot. Unpromoted ints and 810 // floats are passed as right-justified 8-byte values. 811 if (!StackPtr.getNode()) 812 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 813 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 814 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 815 Offset += 4; 816 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 817 DAG.getIntPtrConstant(Offset)); 818 819 // Emit the store. 820 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 821 MachinePointerInfo(), 822 false, false, 0)); 823 } 824 } 825 826 // Join the stores, which are independent of one another. 827 if (!MemOpChains.empty()) 828 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 829 &MemOpChains[0], MemOpChains.size()); 830 831 // Accept direct calls by converting symbolic call addresses to the 832 // associated Target* opcodes. Force %r1 to be used for indirect 833 // tail calls. 834 SDValue Glue; 835 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 836 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 837 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 838 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 839 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 840 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 841 } else if (IsTailCall) { 842 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 843 Glue = Chain.getValue(1); 844 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 845 } 846 847 // Build a sequence of copy-to-reg nodes, chained and glued together. 848 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 849 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 850 RegsToPass[I].second, Glue); 851 Glue = Chain.getValue(1); 852 } 853 854 // The first call operand is the chain and the second is the target address. 855 SmallVector<SDValue, 8> Ops; 856 Ops.push_back(Chain); 857 Ops.push_back(Callee); 858 859 // Add argument registers to the end of the list so that they are 860 // known live into the call. 861 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 862 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 863 RegsToPass[I].second.getValueType())); 864 865 // Glue the call to the argument copies, if any. 866 if (Glue.getNode()) 867 Ops.push_back(Glue); 868 869 // Emit the call. 870 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 871 if (IsTailCall) 872 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, &Ops[0], Ops.size()); 873 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 874 Glue = Chain.getValue(1); 875 876 // Mark the end of the call, which is glued to the call itself. 877 Chain = DAG.getCALLSEQ_END(Chain, 878 DAG.getConstant(NumBytes, PtrVT, true), 879 DAG.getConstant(0, PtrVT, true), 880 Glue, DL); 881 Glue = Chain.getValue(1); 882 883 // Assign locations to each value returned by this call. 884 SmallVector<CCValAssign, 16> RetLocs; 885 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 886 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 887 888 // Copy all of the result registers out of their specified physreg. 889 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 890 CCValAssign &VA = RetLocs[I]; 891 892 // Copy the value out, gluing the copy to the end of the call sequence. 893 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 894 VA.getLocVT(), Glue); 895 Chain = RetValue.getValue(1); 896 Glue = RetValue.getValue(2); 897 898 // Convert the value of the return register into the value that's 899 // being returned. 900 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 901 } 902 903 return Chain; 904} 905 906SDValue 907SystemZTargetLowering::LowerReturn(SDValue Chain, 908 CallingConv::ID CallConv, bool IsVarArg, 909 const SmallVectorImpl<ISD::OutputArg> &Outs, 910 const SmallVectorImpl<SDValue> &OutVals, 911 SDLoc DL, SelectionDAG &DAG) const { 912 MachineFunction &MF = DAG.getMachineFunction(); 913 914 // Assign locations to each returned value. 915 SmallVector<CCValAssign, 16> RetLocs; 916 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 917 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 918 919 // Quick exit for void returns 920 if (RetLocs.empty()) 921 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 922 923 // Copy the result values into the output registers. 924 SDValue Glue; 925 SmallVector<SDValue, 4> RetOps; 926 RetOps.push_back(Chain); 927 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 928 CCValAssign &VA = RetLocs[I]; 929 SDValue RetValue = OutVals[I]; 930 931 // Make the return register live on exit. 932 assert(VA.isRegLoc() && "Can only return in registers!"); 933 934 // Promote the value as required. 935 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 936 937 // Chain and glue the copies together. 938 unsigned Reg = VA.getLocReg(); 939 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 940 Glue = Chain.getValue(1); 941 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 942 } 943 944 // Update chain and glue. 945 RetOps[0] = Chain; 946 if (Glue.getNode()) 947 RetOps.push_back(Glue); 948 949 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 950 RetOps.data(), RetOps.size()); 951} 952 953// CC is a comparison that will be implemented using an integer or 954// floating-point comparison. Return the condition code mask for 955// a branch on true. In the integer case, CCMASK_CMP_UO is set for 956// unsigned comparisons and clear for signed ones. In the floating-point 957// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 958static unsigned CCMaskForCondCode(ISD::CondCode CC) { 959#define CONV(X) \ 960 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 961 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 962 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 963 964 switch (CC) { 965 default: 966 llvm_unreachable("Invalid integer condition!"); 967 968 CONV(EQ); 969 CONV(NE); 970 CONV(GT); 971 CONV(GE); 972 CONV(LT); 973 CONV(LE); 974 975 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 976 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 977 } 978#undef CONV 979} 980 981// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 982// can be converted to a comparison against zero, adjust the operands 983// as necessary. 984static void adjustZeroCmp(SelectionDAG &DAG, bool &IsUnsigned, 985 SDValue &CmpOp0, SDValue &CmpOp1, 986 unsigned &CCMask) { 987 if (IsUnsigned) 988 return; 989 990 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(CmpOp1.getNode()); 991 if (!ConstOp1) 992 return; 993 994 int64_t Value = ConstOp1->getSExtValue(); 995 if ((Value == -1 && CCMask == SystemZ::CCMASK_CMP_GT) || 996 (Value == -1 && CCMask == SystemZ::CCMASK_CMP_LE) || 997 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_LT) || 998 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_GE)) { 999 CCMask ^= SystemZ::CCMASK_CMP_EQ; 1000 CmpOp1 = DAG.getConstant(0, CmpOp1.getValueType()); 1001 } 1002} 1003 1004// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 1005// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 1006static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 1007 SDValue &CmpOp0, SDValue &CmpOp1, 1008 unsigned &CCMask) { 1009 // For us to make any changes, it must a comparison between a single-use 1010 // load and a constant. 1011 if (!CmpOp0.hasOneUse() || 1012 CmpOp0.getOpcode() != ISD::LOAD || 1013 CmpOp1.getOpcode() != ISD::Constant) 1014 return; 1015 1016 // We must have an 8- or 16-bit load. 1017 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 1018 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1019 if (NumBits != 8 && NumBits != 16) 1020 return; 1021 1022 // The load must be an extending one and the constant must be within the 1023 // range of the unextended value. 1024 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 1025 uint64_t Value = Constant->getZExtValue(); 1026 uint64_t Mask = (1 << NumBits) - 1; 1027 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1028 int64_t SignedValue = Constant->getSExtValue(); 1029 if (uint64_t(SignedValue) + (1ULL << (NumBits - 1)) > Mask) 1030 return; 1031 // Unsigned comparison between two sign-extended values is equivalent 1032 // to unsigned comparison between two zero-extended values. 1033 if (IsUnsigned) 1034 Value &= Mask; 1035 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 1036 CCMask == SystemZ::CCMASK_CMP_NE) 1037 // Any choice of IsUnsigned is OK for equality comparisons. 1038 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 1039 // but since we use CLHHSI for zero extensions, it seems better 1040 // to be consistent and do the same here. 1041 Value &= Mask, IsUnsigned = true; 1042 else if (NumBits == 8) { 1043 // Try to treat the comparison as unsigned, so that we can use CLI. 1044 // Adjust CCMask and Value as necessary. 1045 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 1046 // Test whether the high bit of the byte is set. 1047 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 1048 else if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_GE) 1049 // Test whether the high bit of the byte is clear. 1050 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 1051 else 1052 // No instruction exists for this combination. 1053 return; 1054 } 1055 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1056 if (Value > Mask) 1057 return; 1058 // Signed comparison between two zero-extended values is equivalent 1059 // to unsigned comparison. 1060 IsUnsigned = true; 1061 } else 1062 return; 1063 1064 // Make sure that the first operand is an i32 of the right extension type. 1065 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 1066 if (CmpOp0.getValueType() != MVT::i32 || 1067 Load->getExtensionType() != ExtType) 1068 CmpOp0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1069 Load->getChain(), Load->getBasePtr(), 1070 Load->getPointerInfo(), Load->getMemoryVT(), 1071 Load->isVolatile(), Load->isNonTemporal(), 1072 Load->getAlignment()); 1073 1074 // Make sure that the second operand is an i32 with the right value. 1075 if (CmpOp1.getValueType() != MVT::i32 || 1076 Value != Constant->getZExtValue()) 1077 CmpOp1 = DAG.getConstant(Value, MVT::i32); 1078} 1079 1080// Return true if Op is either an unextended load, or a load suitable 1081// for integer register-memory comparisons of type ICmpType. 1082static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 1083 LoadSDNode *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1084 if (Load) { 1085 // There are no instructions to compare a register with a memory byte. 1086 if (Load->getMemoryVT() == MVT::i8) 1087 return false; 1088 // Otherwise decide on extension type. 1089 switch (Load->getExtensionType()) { 1090 case ISD::NON_EXTLOAD: 1091 return true; 1092 case ISD::SEXTLOAD: 1093 return ICmpType != SystemZICMP::UnsignedOnly; 1094 case ISD::ZEXTLOAD: 1095 return ICmpType != SystemZICMP::SignedOnly; 1096 default: 1097 break; 1098 } 1099 } 1100 return false; 1101} 1102 1103// Return true if it is better to swap comparison operands Op0 and Op1. 1104// ICmpType is the type of an integer comparison. 1105static bool shouldSwapCmpOperands(SDValue Op0, SDValue Op1, 1106 unsigned ICmpType) { 1107 // Leave f128 comparisons alone, since they have no memory forms. 1108 if (Op0.getValueType() == MVT::f128) 1109 return false; 1110 1111 // Always keep a floating-point constant second, since comparisons with 1112 // zero can use LOAD TEST and comparisons with other constants make a 1113 // natural memory operand. 1114 if (isa<ConstantFPSDNode>(Op1)) 1115 return false; 1116 1117 // Never swap comparisons with zero since there are many ways to optimize 1118 // those later. 1119 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1); 1120 if (COp1 && COp1->getZExtValue() == 0) 1121 return false; 1122 1123 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1124 // In that case we generally prefer the memory to be second. 1125 if ((isNaturalMemoryOperand(Op0, ICmpType) && Op0.hasOneUse()) && 1126 !(isNaturalMemoryOperand(Op1, ICmpType) && Op1.hasOneUse())) { 1127 // The only exceptions are when the second operand is a constant and 1128 // we can use things like CHHSI. 1129 if (!COp1) 1130 return true; 1131 // The unsigned memory-immediate instructions can handle 16-bit 1132 // unsigned integers. 1133 if (ICmpType != SystemZICMP::SignedOnly && 1134 isUInt<16>(COp1->getZExtValue())) 1135 return false; 1136 // The signed memory-immediate instructions can handle 16-bit 1137 // signed integers. 1138 if (ICmpType != SystemZICMP::UnsignedOnly && 1139 isInt<16>(COp1->getSExtValue())) 1140 return false; 1141 return true; 1142 } 1143 return false; 1144} 1145 1146// Return true if shift operation N has an in-range constant shift value. 1147// Store it in ShiftVal if so. 1148static bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 1149 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1150 if (!Shift) 1151 return false; 1152 1153 uint64_t Amount = Shift->getZExtValue(); 1154 if (Amount >= N.getValueType().getSizeInBits()) 1155 return false; 1156 1157 ShiftVal = Amount; 1158 return true; 1159} 1160 1161// Check whether an AND with Mask is suitable for a TEST UNDER MASK 1162// instruction and whether the CC value is descriptive enough to handle 1163// a comparison of type Opcode between the AND result and CmpVal. 1164// CCMask says which comparison result is being tested and BitSize is 1165// the number of bits in the operands. If TEST UNDER MASK can be used, 1166// return the corresponding CC mask, otherwise return 0. 1167static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 1168 uint64_t Mask, uint64_t CmpVal, 1169 unsigned ICmpType) { 1170 assert(Mask != 0 && "ANDs with zero should have been removed by now"); 1171 1172 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 1173 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 1174 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 1175 return 0; 1176 1177 // Work out the masks for the lowest and highest bits. 1178 unsigned HighShift = 63 - countLeadingZeros(Mask); 1179 uint64_t High = uint64_t(1) << HighShift; 1180 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 1181 1182 // Signed ordered comparisons are effectively unsigned if the sign 1183 // bit is dropped. 1184 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 1185 1186 // Check for equality comparisons with 0, or the equivalent. 1187 if (CmpVal == 0) { 1188 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1189 return SystemZ::CCMASK_TM_ALL_0; 1190 if (CCMask == SystemZ::CCMASK_CMP_NE) 1191 return SystemZ::CCMASK_TM_SOME_1; 1192 } 1193 if (EffectivelyUnsigned && CmpVal <= Low) { 1194 if (CCMask == SystemZ::CCMASK_CMP_LT) 1195 return SystemZ::CCMASK_TM_ALL_0; 1196 if (CCMask == SystemZ::CCMASK_CMP_GE) 1197 return SystemZ::CCMASK_TM_SOME_1; 1198 } 1199 if (EffectivelyUnsigned && CmpVal < Low) { 1200 if (CCMask == SystemZ::CCMASK_CMP_LE) 1201 return SystemZ::CCMASK_TM_ALL_0; 1202 if (CCMask == SystemZ::CCMASK_CMP_GT) 1203 return SystemZ::CCMASK_TM_SOME_1; 1204 } 1205 1206 // Check for equality comparisons with the mask, or the equivalent. 1207 if (CmpVal == Mask) { 1208 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1209 return SystemZ::CCMASK_TM_ALL_1; 1210 if (CCMask == SystemZ::CCMASK_CMP_NE) 1211 return SystemZ::CCMASK_TM_SOME_0; 1212 } 1213 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 1214 if (CCMask == SystemZ::CCMASK_CMP_GT) 1215 return SystemZ::CCMASK_TM_ALL_1; 1216 if (CCMask == SystemZ::CCMASK_CMP_LE) 1217 return SystemZ::CCMASK_TM_SOME_0; 1218 } 1219 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 1220 if (CCMask == SystemZ::CCMASK_CMP_GE) 1221 return SystemZ::CCMASK_TM_ALL_1; 1222 if (CCMask == SystemZ::CCMASK_CMP_LT) 1223 return SystemZ::CCMASK_TM_SOME_0; 1224 } 1225 1226 // Check for ordered comparisons with the top bit. 1227 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 1228 if (CCMask == SystemZ::CCMASK_CMP_LE) 1229 return SystemZ::CCMASK_TM_MSB_0; 1230 if (CCMask == SystemZ::CCMASK_CMP_GT) 1231 return SystemZ::CCMASK_TM_MSB_1; 1232 } 1233 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 1234 if (CCMask == SystemZ::CCMASK_CMP_LT) 1235 return SystemZ::CCMASK_TM_MSB_0; 1236 if (CCMask == SystemZ::CCMASK_CMP_GE) 1237 return SystemZ::CCMASK_TM_MSB_1; 1238 } 1239 1240 // If there are just two bits, we can do equality checks for Low and High 1241 // as well. 1242 if (Mask == Low + High) { 1243 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 1244 return SystemZ::CCMASK_TM_MIXED_MSB_0; 1245 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 1246 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 1247 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 1248 return SystemZ::CCMASK_TM_MIXED_MSB_1; 1249 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 1250 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 1251 } 1252 1253 // Looks like we've exhausted our options. 1254 return 0; 1255} 1256 1257// See whether the comparison (Opcode CmpOp0, CmpOp1, ICmpType) can be 1258// implemented as a TEST UNDER MASK instruction when the condition being 1259// tested is as described by CCValid and CCMask. Update the arguments 1260// with the TM version if so. 1261static void adjustForTestUnderMask(SelectionDAG &DAG, unsigned &Opcode, 1262 SDValue &CmpOp0, SDValue &CmpOp1, 1263 unsigned &CCValid, unsigned &CCMask, 1264 unsigned &ICmpType) { 1265 // Check that we have a comparison with a constant. 1266 ConstantSDNode *ConstCmpOp1 = dyn_cast<ConstantSDNode>(CmpOp1); 1267 if (!ConstCmpOp1) 1268 return; 1269 uint64_t CmpVal = ConstCmpOp1->getZExtValue(); 1270 1271 // Check whether the nonconstant input is an AND with a constant mask. 1272 if (CmpOp0.getOpcode() != ISD::AND) 1273 return; 1274 SDValue AndOp0 = CmpOp0.getOperand(0); 1275 SDValue AndOp1 = CmpOp0.getOperand(1); 1276 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(AndOp1.getNode()); 1277 if (!Mask) 1278 return; 1279 uint64_t MaskVal = Mask->getZExtValue(); 1280 1281 // Check whether the combination of mask, comparison value and comparison 1282 // type are suitable. 1283 unsigned BitSize = CmpOp0.getValueType().getSizeInBits(); 1284 unsigned NewCCMask, ShiftVal; 1285 if (ICmpType != SystemZICMP::SignedOnly && 1286 AndOp0.getOpcode() == ISD::SHL && 1287 isSimpleShift(AndOp0, ShiftVal) && 1288 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask, MaskVal >> ShiftVal, 1289 CmpVal >> ShiftVal, 1290 SystemZICMP::Any))) { 1291 AndOp0 = AndOp0.getOperand(0); 1292 AndOp1 = DAG.getConstant(MaskVal >> ShiftVal, AndOp0.getValueType()); 1293 } else if (ICmpType != SystemZICMP::SignedOnly && 1294 AndOp0.getOpcode() == ISD::SRL && 1295 isSimpleShift(AndOp0, ShiftVal) && 1296 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask, 1297 MaskVal << ShiftVal, 1298 CmpVal << ShiftVal, 1299 SystemZICMP::UnsignedOnly))) { 1300 AndOp0 = AndOp0.getOperand(0); 1301 AndOp1 = DAG.getConstant(MaskVal << ShiftVal, AndOp0.getValueType()); 1302 } else { 1303 NewCCMask = getTestUnderMaskCond(BitSize, CCMask, MaskVal, CmpVal, 1304 ICmpType); 1305 if (!NewCCMask) 1306 return; 1307 } 1308 1309 // Go ahead and make the change. 1310 Opcode = SystemZISD::TM; 1311 CmpOp0 = AndOp0; 1312 CmpOp1 = AndOp1; 1313 ICmpType = (bool(NewCCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 1314 bool(NewCCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 1315 CCValid = SystemZ::CCMASK_TM; 1316 CCMask = NewCCMask; 1317} 1318 1319// Return a target node that compares CmpOp0 with CmpOp1 and stores a 1320// 2-bit result in CC. Set CCValid to the CCMASK_* of all possible 1321// 2-bit results and CCMask to the subset of those results that are 1322// associated with Cond. 1323static SDValue emitCmp(const SystemZTargetMachine &TM, SelectionDAG &DAG, 1324 SDLoc DL, SDValue CmpOp0, SDValue CmpOp1, 1325 ISD::CondCode Cond, unsigned &CCValid, 1326 unsigned &CCMask) { 1327 bool IsUnsigned = false; 1328 CCMask = CCMaskForCondCode(Cond); 1329 unsigned Opcode, ICmpType = 0; 1330 if (CmpOp0.getValueType().isFloatingPoint()) { 1331 CCValid = SystemZ::CCMASK_FCMP; 1332 Opcode = SystemZISD::FCMP; 1333 } else { 1334 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 1335 CCValid = SystemZ::CCMASK_ICMP; 1336 CCMask &= CCValid; 1337 adjustZeroCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1338 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1339 Opcode = SystemZISD::ICMP; 1340 // Choose the type of comparison. Equality and inequality tests can 1341 // use either signed or unsigned comparisons. The choice also doesn't 1342 // matter if both sign bits are known to be clear. In those cases we 1343 // want to give the main isel code the freedom to choose whichever 1344 // form fits best. 1345 if (CCMask == SystemZ::CCMASK_CMP_EQ || 1346 CCMask == SystemZ::CCMASK_CMP_NE || 1347 (DAG.SignBitIsZero(CmpOp0) && DAG.SignBitIsZero(CmpOp1))) 1348 ICmpType = SystemZICMP::Any; 1349 else if (IsUnsigned) 1350 ICmpType = SystemZICMP::UnsignedOnly; 1351 else 1352 ICmpType = SystemZICMP::SignedOnly; 1353 } 1354 1355 if (shouldSwapCmpOperands(CmpOp0, CmpOp1, ICmpType)) { 1356 std::swap(CmpOp0, CmpOp1); 1357 CCMask = ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1358 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1359 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1360 (CCMask & SystemZ::CCMASK_CMP_UO)); 1361 } 1362 1363 adjustForTestUnderMask(DAG, Opcode, CmpOp0, CmpOp1, CCValid, CCMask, 1364 ICmpType); 1365 if (Opcode == SystemZISD::ICMP || Opcode == SystemZISD::TM) 1366 return DAG.getNode(Opcode, DL, MVT::Glue, CmpOp0, CmpOp1, 1367 DAG.getConstant(ICmpType, MVT::i32)); 1368 return DAG.getNode(Opcode, DL, MVT::Glue, CmpOp0, CmpOp1); 1369} 1370 1371// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1372// 64 bits. Extend is the extension type to use. Store the high part 1373// in Hi and the low part in Lo. 1374static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1375 unsigned Extend, SDValue Op0, SDValue Op1, 1376 SDValue &Hi, SDValue &Lo) { 1377 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1378 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1379 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1380 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1381 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1382 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1383} 1384 1385// Lower a binary operation that produces two VT results, one in each 1386// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1387// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1388// on the extended Op0 and (unextended) Op1. Store the even register result 1389// in Even and the odd register result in Odd. 1390static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1391 unsigned Extend, unsigned Opcode, 1392 SDValue Op0, SDValue Op1, 1393 SDValue &Even, SDValue &Odd) { 1394 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1395 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1396 SDValue(In128, 0), Op1); 1397 bool Is32Bit = is32Bit(VT); 1398 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 1399 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 1400} 1401 1402SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1403 SDValue Chain = Op.getOperand(0); 1404 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1405 SDValue CmpOp0 = Op.getOperand(2); 1406 SDValue CmpOp1 = Op.getOperand(3); 1407 SDValue Dest = Op.getOperand(4); 1408 SDLoc DL(Op); 1409 1410 unsigned CCValid, CCMask; 1411 SDValue Flags = emitCmp(TM, DAG, DL, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1412 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1413 Chain, DAG.getConstant(CCValid, MVT::i32), 1414 DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 1415} 1416 1417SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1418 SelectionDAG &DAG) const { 1419 SDValue CmpOp0 = Op.getOperand(0); 1420 SDValue CmpOp1 = Op.getOperand(1); 1421 SDValue TrueOp = Op.getOperand(2); 1422 SDValue FalseOp = Op.getOperand(3); 1423 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1424 SDLoc DL(Op); 1425 1426 unsigned CCValid, CCMask; 1427 SDValue Flags = emitCmp(TM, DAG, DL, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1428 1429 SmallVector<SDValue, 5> Ops; 1430 Ops.push_back(TrueOp); 1431 Ops.push_back(FalseOp); 1432 Ops.push_back(DAG.getConstant(CCValid, MVT::i32)); 1433 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1434 Ops.push_back(Flags); 1435 1436 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1437 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1438} 1439 1440SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1441 SelectionDAG &DAG) const { 1442 SDLoc DL(Node); 1443 const GlobalValue *GV = Node->getGlobal(); 1444 int64_t Offset = Node->getOffset(); 1445 EVT PtrVT = getPointerTy(); 1446 Reloc::Model RM = TM.getRelocationModel(); 1447 CodeModel::Model CM = TM.getCodeModel(); 1448 1449 SDValue Result; 1450 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1451 // Assign anchors at 1<<12 byte boundaries. 1452 uint64_t Anchor = Offset & ~uint64_t(0xfff); 1453 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 1454 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1455 1456 // The offset can be folded into the address if it is aligned to a halfword. 1457 Offset -= Anchor; 1458 if (Offset != 0 && (Offset & 1) == 0) { 1459 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 1460 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 1461 Offset = 0; 1462 } 1463 } else { 1464 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1465 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1466 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1467 MachinePointerInfo::getGOT(), false, false, false, 0); 1468 } 1469 1470 // If there was a non-zero offset that we didn't fold, create an explicit 1471 // addition for it. 1472 if (Offset != 0) 1473 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1474 DAG.getConstant(Offset, PtrVT)); 1475 1476 return Result; 1477} 1478 1479SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1480 SelectionDAG &DAG) const { 1481 SDLoc DL(Node); 1482 const GlobalValue *GV = Node->getGlobal(); 1483 EVT PtrVT = getPointerTy(); 1484 TLSModel::Model model = TM.getTLSModel(GV); 1485 1486 if (model != TLSModel::LocalExec) 1487 llvm_unreachable("only local-exec TLS mode supported"); 1488 1489 // The high part of the thread pointer is in access register 0. 1490 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1491 DAG.getConstant(0, MVT::i32)); 1492 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1493 1494 // The low part of the thread pointer is in access register 1. 1495 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1496 DAG.getConstant(1, MVT::i32)); 1497 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1498 1499 // Merge them into a single 64-bit address. 1500 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1501 DAG.getConstant(32, PtrVT)); 1502 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1503 1504 // Get the offset of GA from the thread pointer. 1505 SystemZConstantPoolValue *CPV = 1506 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1507 1508 // Force the offset into the constant pool and load it from there. 1509 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1510 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1511 CPAddr, MachinePointerInfo::getConstantPool(), 1512 false, false, false, 0); 1513 1514 // Add the base and offset together. 1515 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1516} 1517 1518SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1519 SelectionDAG &DAG) const { 1520 SDLoc DL(Node); 1521 const BlockAddress *BA = Node->getBlockAddress(); 1522 int64_t Offset = Node->getOffset(); 1523 EVT PtrVT = getPointerTy(); 1524 1525 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1526 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1527 return Result; 1528} 1529 1530SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1531 SelectionDAG &DAG) const { 1532 SDLoc DL(JT); 1533 EVT PtrVT = getPointerTy(); 1534 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1535 1536 // Use LARL to load the address of the table. 1537 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1538} 1539 1540SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1541 SelectionDAG &DAG) const { 1542 SDLoc DL(CP); 1543 EVT PtrVT = getPointerTy(); 1544 1545 SDValue Result; 1546 if (CP->isMachineConstantPoolEntry()) 1547 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1548 CP->getAlignment()); 1549 else 1550 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1551 CP->getAlignment(), CP->getOffset()); 1552 1553 // Use LARL to load the address of the constant pool entry. 1554 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1555} 1556 1557SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1558 SelectionDAG &DAG) const { 1559 SDLoc DL(Op); 1560 SDValue In = Op.getOperand(0); 1561 EVT InVT = In.getValueType(); 1562 EVT ResVT = Op.getValueType(); 1563 1564 SDValue Shift32 = DAG.getConstant(32, MVT::i64); 1565 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1566 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1567 SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); 1568 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); 1569 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, 1570 DL, MVT::f32, Out64); 1571 } 1572 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1573 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1574 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1575 MVT::f64, SDValue(U64, 0), In); 1576 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 1577 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); 1578 SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1579 return Out; 1580 } 1581 llvm_unreachable("Unexpected bitcast combination"); 1582} 1583 1584SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1585 SelectionDAG &DAG) const { 1586 MachineFunction &MF = DAG.getMachineFunction(); 1587 SystemZMachineFunctionInfo *FuncInfo = 1588 MF.getInfo<SystemZMachineFunctionInfo>(); 1589 EVT PtrVT = getPointerTy(); 1590 1591 SDValue Chain = Op.getOperand(0); 1592 SDValue Addr = Op.getOperand(1); 1593 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1594 SDLoc DL(Op); 1595 1596 // The initial values of each field. 1597 const unsigned NumFields = 4; 1598 SDValue Fields[NumFields] = { 1599 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1600 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1601 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1602 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1603 }; 1604 1605 // Store each field into its respective slot. 1606 SDValue MemOps[NumFields]; 1607 unsigned Offset = 0; 1608 for (unsigned I = 0; I < NumFields; ++I) { 1609 SDValue FieldAddr = Addr; 1610 if (Offset != 0) 1611 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1612 DAG.getIntPtrConstant(Offset)); 1613 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1614 MachinePointerInfo(SV, Offset), 1615 false, false, 0); 1616 Offset += 8; 1617 } 1618 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1619} 1620 1621SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1622 SelectionDAG &DAG) const { 1623 SDValue Chain = Op.getOperand(0); 1624 SDValue DstPtr = Op.getOperand(1); 1625 SDValue SrcPtr = Op.getOperand(2); 1626 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1627 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1628 SDLoc DL(Op); 1629 1630 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1631 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1632 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1633} 1634 1635SDValue SystemZTargetLowering:: 1636lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1637 SDValue Chain = Op.getOperand(0); 1638 SDValue Size = Op.getOperand(1); 1639 SDLoc DL(Op); 1640 1641 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1642 1643 // Get a reference to the stack pointer. 1644 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1645 1646 // Get the new stack pointer value. 1647 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1648 1649 // Copy the new stack pointer back. 1650 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1651 1652 // The allocated data lives above the 160 bytes allocated for the standard 1653 // frame, plus any outgoing stack arguments. We don't know how much that 1654 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1655 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1656 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1657 1658 SDValue Ops[2] = { Result, Chain }; 1659 return DAG.getMergeValues(Ops, 2, DL); 1660} 1661 1662SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 1663 SelectionDAG &DAG) const { 1664 EVT VT = Op.getValueType(); 1665 SDLoc DL(Op); 1666 SDValue Ops[2]; 1667 if (is32Bit(VT)) 1668 // Just do a normal 64-bit multiplication and extract the results. 1669 // We define this so that it can be used for constant division. 1670 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 1671 Op.getOperand(1), Ops[1], Ops[0]); 1672 else { 1673 // Do a full 128-bit multiplication based on UMUL_LOHI64: 1674 // 1675 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 1676 // 1677 // but using the fact that the upper halves are either all zeros 1678 // or all ones: 1679 // 1680 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 1681 // 1682 // and grouping the right terms together since they are quicker than the 1683 // multiplication: 1684 // 1685 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 1686 SDValue C63 = DAG.getConstant(63, MVT::i64); 1687 SDValue LL = Op.getOperand(0); 1688 SDValue RL = Op.getOperand(1); 1689 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 1690 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 1691 // UMUL_LOHI64 returns the low result in the odd register and the high 1692 // result in the even register. SMUL_LOHI is defined to return the 1693 // low half first, so the results are in reverse order. 1694 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1695 LL, RL, Ops[1], Ops[0]); 1696 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 1697 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 1698 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 1699 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 1700 } 1701 return DAG.getMergeValues(Ops, 2, DL); 1702} 1703 1704SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1705 SelectionDAG &DAG) const { 1706 EVT VT = Op.getValueType(); 1707 SDLoc DL(Op); 1708 SDValue Ops[2]; 1709 if (is32Bit(VT)) 1710 // Just do a normal 64-bit multiplication and extract the results. 1711 // We define this so that it can be used for constant division. 1712 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 1713 Op.getOperand(1), Ops[1], Ops[0]); 1714 else 1715 // UMUL_LOHI64 returns the low result in the odd register and the high 1716 // result in the even register. UMUL_LOHI is defined to return the 1717 // low half first, so the results are in reverse order. 1718 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1719 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1720 return DAG.getMergeValues(Ops, 2, DL); 1721} 1722 1723SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1724 SelectionDAG &DAG) const { 1725 SDValue Op0 = Op.getOperand(0); 1726 SDValue Op1 = Op.getOperand(1); 1727 EVT VT = Op.getValueType(); 1728 SDLoc DL(Op); 1729 unsigned Opcode; 1730 1731 // We use DSGF for 32-bit division. 1732 if (is32Bit(VT)) { 1733 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1734 Opcode = SystemZISD::SDIVREM32; 1735 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 1736 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 1737 Opcode = SystemZISD::SDIVREM32; 1738 } else 1739 Opcode = SystemZISD::SDIVREM64; 1740 1741 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1742 // input is "don't care". The instruction returns the remainder in 1743 // the even register and the quotient in the odd register. 1744 SDValue Ops[2]; 1745 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 1746 Op0, Op1, Ops[1], Ops[0]); 1747 return DAG.getMergeValues(Ops, 2, DL); 1748} 1749 1750SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1751 SelectionDAG &DAG) const { 1752 EVT VT = Op.getValueType(); 1753 SDLoc DL(Op); 1754 1755 // DL(G) uses a double-width dividend, so we need to clear the even 1756 // register in the GR128 input. The instruction returns the remainder 1757 // in the even register and the quotient in the odd register. 1758 SDValue Ops[2]; 1759 if (is32Bit(VT)) 1760 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1761 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1762 else 1763 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1764 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1765 return DAG.getMergeValues(Ops, 2, DL); 1766} 1767 1768SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1769 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1770 1771 // Get the known-zero masks for each operand. 1772 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1773 APInt KnownZero[2], KnownOne[2]; 1774 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1775 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1776 1777 // See if the upper 32 bits of one operand and the lower 32 bits of the 1778 // other are known zero. They are the low and high operands respectively. 1779 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1780 KnownZero[1].getZExtValue() }; 1781 unsigned High, Low; 1782 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1783 High = 1, Low = 0; 1784 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1785 High = 0, Low = 1; 1786 else 1787 return Op; 1788 1789 SDValue LowOp = Ops[Low]; 1790 SDValue HighOp = Ops[High]; 1791 1792 // If the high part is a constant, we're better off using IILH. 1793 if (HighOp.getOpcode() == ISD::Constant) 1794 return Op; 1795 1796 // If the low part is a constant that is outside the range of LHI, 1797 // then we're better off using IILF. 1798 if (LowOp.getOpcode() == ISD::Constant) { 1799 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1800 if (!isInt<16>(Value)) 1801 return Op; 1802 } 1803 1804 // Check whether the high part is an AND that doesn't change the 1805 // high 32 bits and just masks out low bits. We can skip it if so. 1806 if (HighOp.getOpcode() == ISD::AND && 1807 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1808 ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); 1809 uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; 1810 if ((Mask >> 32) == 0xffffffff) 1811 HighOp = HighOp.getOperand(0); 1812 } 1813 1814 // Take advantage of the fact that all GR32 operations only change the 1815 // low 32 bits by truncating Low to an i32 and inserting it directly 1816 // using a subreg. The interesting cases are those where the truncation 1817 // can be folded. 1818 SDLoc DL(Op); 1819 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1820 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, 1821 MVT::i64, HighOp, Low32); 1822} 1823 1824// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1825// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1826SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1827 SelectionDAG &DAG, 1828 unsigned Opcode) const { 1829 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1830 1831 // 32-bit operations need no code outside the main loop. 1832 EVT NarrowVT = Node->getMemoryVT(); 1833 EVT WideVT = MVT::i32; 1834 if (NarrowVT == WideVT) 1835 return Op; 1836 1837 int64_t BitSize = NarrowVT.getSizeInBits(); 1838 SDValue ChainIn = Node->getChain(); 1839 SDValue Addr = Node->getBasePtr(); 1840 SDValue Src2 = Node->getVal(); 1841 MachineMemOperand *MMO = Node->getMemOperand(); 1842 SDLoc DL(Node); 1843 EVT PtrVT = Addr.getValueType(); 1844 1845 // Convert atomic subtracts of constants into additions. 1846 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1847 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1848 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1849 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1850 } 1851 1852 // Get the address of the containing word. 1853 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1854 DAG.getConstant(-4, PtrVT)); 1855 1856 // Get the number of bits that the word must be rotated left in order 1857 // to bring the field to the top bits of a GR32. 1858 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1859 DAG.getConstant(3, PtrVT)); 1860 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1861 1862 // Get the complementing shift amount, for rotating a field in the top 1863 // bits back to its proper position. 1864 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1865 DAG.getConstant(0, WideVT), BitShift); 1866 1867 // Extend the source operand to 32 bits and prepare it for the inner loop. 1868 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1869 // operations require the source to be shifted in advance. (This shift 1870 // can be folded if the source is constant.) For AND and NAND, the lower 1871 // bits must be set, while for other opcodes they should be left clear. 1872 if (Opcode != SystemZISD::ATOMIC_SWAPW) 1873 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 1874 DAG.getConstant(32 - BitSize, WideVT)); 1875 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 1876 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 1877 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 1878 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 1879 1880 // Construct the ATOMIC_LOADW_* node. 1881 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1882 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 1883 DAG.getConstant(BitSize, WideVT) }; 1884 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 1885 array_lengthof(Ops), 1886 NarrowVT, MMO); 1887 1888 // Rotate the result of the final CS so that the field is in the lower 1889 // bits of a GR32, then truncate it. 1890 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 1891 DAG.getConstant(BitSize, WideVT)); 1892 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 1893 1894 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 1895 return DAG.getMergeValues(RetOps, 2, DL); 1896} 1897 1898// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 1899// into a fullword ATOMIC_CMP_SWAPW operation. 1900SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 1901 SelectionDAG &DAG) const { 1902 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1903 1904 // We have native support for 32-bit compare and swap. 1905 EVT NarrowVT = Node->getMemoryVT(); 1906 EVT WideVT = MVT::i32; 1907 if (NarrowVT == WideVT) 1908 return Op; 1909 1910 int64_t BitSize = NarrowVT.getSizeInBits(); 1911 SDValue ChainIn = Node->getOperand(0); 1912 SDValue Addr = Node->getOperand(1); 1913 SDValue CmpVal = Node->getOperand(2); 1914 SDValue SwapVal = Node->getOperand(3); 1915 MachineMemOperand *MMO = Node->getMemOperand(); 1916 SDLoc DL(Node); 1917 EVT PtrVT = Addr.getValueType(); 1918 1919 // Get the address of the containing word. 1920 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1921 DAG.getConstant(-4, PtrVT)); 1922 1923 // Get the number of bits that the word must be rotated left in order 1924 // to bring the field to the top bits of a GR32. 1925 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1926 DAG.getConstant(3, PtrVT)); 1927 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1928 1929 // Get the complementing shift amount, for rotating a field in the top 1930 // bits back to its proper position. 1931 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1932 DAG.getConstant(0, WideVT), BitShift); 1933 1934 // Construct the ATOMIC_CMP_SWAPW node. 1935 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1936 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 1937 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 1938 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 1939 VTList, Ops, array_lengthof(Ops), 1940 NarrowVT, MMO); 1941 return AtomicOp; 1942} 1943 1944SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 1945 SelectionDAG &DAG) const { 1946 MachineFunction &MF = DAG.getMachineFunction(); 1947 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1948 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 1949 SystemZ::R15D, Op.getValueType()); 1950} 1951 1952SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 1953 SelectionDAG &DAG) const { 1954 MachineFunction &MF = DAG.getMachineFunction(); 1955 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1956 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 1957 SystemZ::R15D, Op.getOperand(1)); 1958} 1959 1960SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op, 1961 SelectionDAG &DAG) const { 1962 bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 1963 if (!IsData) 1964 // Just preserve the chain. 1965 return Op.getOperand(0); 1966 1967 bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 1968 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ; 1969 MemIntrinsicSDNode *Node = cast<MemIntrinsicSDNode>(Op.getNode()); 1970 SDValue Ops[] = { 1971 Op.getOperand(0), 1972 DAG.getConstant(Code, MVT::i32), 1973 Op.getOperand(1) 1974 }; 1975 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, SDLoc(Op), 1976 Node->getVTList(), Ops, array_lengthof(Ops), 1977 Node->getMemoryVT(), Node->getMemOperand()); 1978} 1979 1980SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 1981 SelectionDAG &DAG) const { 1982 switch (Op.getOpcode()) { 1983 case ISD::BR_CC: 1984 return lowerBR_CC(Op, DAG); 1985 case ISD::SELECT_CC: 1986 return lowerSELECT_CC(Op, DAG); 1987 case ISD::GlobalAddress: 1988 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 1989 case ISD::GlobalTLSAddress: 1990 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 1991 case ISD::BlockAddress: 1992 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 1993 case ISD::JumpTable: 1994 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 1995 case ISD::ConstantPool: 1996 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 1997 case ISD::BITCAST: 1998 return lowerBITCAST(Op, DAG); 1999 case ISD::VASTART: 2000 return lowerVASTART(Op, DAG); 2001 case ISD::VACOPY: 2002 return lowerVACOPY(Op, DAG); 2003 case ISD::DYNAMIC_STACKALLOC: 2004 return lowerDYNAMIC_STACKALLOC(Op, DAG); 2005 case ISD::SMUL_LOHI: 2006 return lowerSMUL_LOHI(Op, DAG); 2007 case ISD::UMUL_LOHI: 2008 return lowerUMUL_LOHI(Op, DAG); 2009 case ISD::SDIVREM: 2010 return lowerSDIVREM(Op, DAG); 2011 case ISD::UDIVREM: 2012 return lowerUDIVREM(Op, DAG); 2013 case ISD::OR: 2014 return lowerOR(Op, DAG); 2015 case ISD::ATOMIC_SWAP: 2016 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 2017 case ISD::ATOMIC_LOAD_ADD: 2018 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 2019 case ISD::ATOMIC_LOAD_SUB: 2020 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 2021 case ISD::ATOMIC_LOAD_AND: 2022 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 2023 case ISD::ATOMIC_LOAD_OR: 2024 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 2025 case ISD::ATOMIC_LOAD_XOR: 2026 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 2027 case ISD::ATOMIC_LOAD_NAND: 2028 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 2029 case ISD::ATOMIC_LOAD_MIN: 2030 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 2031 case ISD::ATOMIC_LOAD_MAX: 2032 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 2033 case ISD::ATOMIC_LOAD_UMIN: 2034 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 2035 case ISD::ATOMIC_LOAD_UMAX: 2036 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 2037 case ISD::ATOMIC_CMP_SWAP: 2038 return lowerATOMIC_CMP_SWAP(Op, DAG); 2039 case ISD::STACKSAVE: 2040 return lowerSTACKSAVE(Op, DAG); 2041 case ISD::STACKRESTORE: 2042 return lowerSTACKRESTORE(Op, DAG); 2043 case ISD::PREFETCH: 2044 return lowerPREFETCH(Op, DAG); 2045 default: 2046 llvm_unreachable("Unexpected node to lower"); 2047 } 2048} 2049 2050const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 2051#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 2052 switch (Opcode) { 2053 OPCODE(RET_FLAG); 2054 OPCODE(CALL); 2055 OPCODE(SIBCALL); 2056 OPCODE(PCREL_WRAPPER); 2057 OPCODE(PCREL_OFFSET); 2058 OPCODE(ICMP); 2059 OPCODE(FCMP); 2060 OPCODE(TM); 2061 OPCODE(BR_CCMASK); 2062 OPCODE(SELECT_CCMASK); 2063 OPCODE(ADJDYNALLOC); 2064 OPCODE(EXTRACT_ACCESS); 2065 OPCODE(UMUL_LOHI64); 2066 OPCODE(SDIVREM64); 2067 OPCODE(UDIVREM32); 2068 OPCODE(UDIVREM64); 2069 OPCODE(MVC); 2070 OPCODE(MVC_LOOP); 2071 OPCODE(NC); 2072 OPCODE(NC_LOOP); 2073 OPCODE(OC); 2074 OPCODE(OC_LOOP); 2075 OPCODE(XC); 2076 OPCODE(XC_LOOP); 2077 OPCODE(CLC); 2078 OPCODE(CLC_LOOP); 2079 OPCODE(STRCMP); 2080 OPCODE(STPCPY); 2081 OPCODE(SEARCH_STRING); 2082 OPCODE(IPM); 2083 OPCODE(ATOMIC_SWAPW); 2084 OPCODE(ATOMIC_LOADW_ADD); 2085 OPCODE(ATOMIC_LOADW_SUB); 2086 OPCODE(ATOMIC_LOADW_AND); 2087 OPCODE(ATOMIC_LOADW_OR); 2088 OPCODE(ATOMIC_LOADW_XOR); 2089 OPCODE(ATOMIC_LOADW_NAND); 2090 OPCODE(ATOMIC_LOADW_MIN); 2091 OPCODE(ATOMIC_LOADW_MAX); 2092 OPCODE(ATOMIC_LOADW_UMIN); 2093 OPCODE(ATOMIC_LOADW_UMAX); 2094 OPCODE(ATOMIC_CMP_SWAPW); 2095 OPCODE(PREFETCH); 2096 } 2097 return NULL; 2098#undef OPCODE 2099} 2100 2101//===----------------------------------------------------------------------===// 2102// Custom insertion 2103//===----------------------------------------------------------------------===// 2104 2105// Create a new basic block after MBB. 2106static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 2107 MachineFunction &MF = *MBB->getParent(); 2108 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 2109 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 2110 return NewMBB; 2111} 2112 2113// Split MBB after MI and return the new block (the one that contains 2114// instructions after MI). 2115static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 2116 MachineBasicBlock *MBB) { 2117 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2118 NewMBB->splice(NewMBB->begin(), MBB, 2119 llvm::next(MachineBasicBlock::iterator(MI)), 2120 MBB->end()); 2121 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2122 return NewMBB; 2123} 2124 2125// Split MBB before MI and return the new block (the one that contains MI). 2126static MachineBasicBlock *splitBlockBefore(MachineInstr *MI, 2127 MachineBasicBlock *MBB) { 2128 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2129 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 2130 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2131 return NewMBB; 2132} 2133 2134// Force base value Base into a register before MI. Return the register. 2135static unsigned forceReg(MachineInstr *MI, MachineOperand &Base, 2136 const SystemZInstrInfo *TII) { 2137 if (Base.isReg()) 2138 return Base.getReg(); 2139 2140 MachineBasicBlock *MBB = MI->getParent(); 2141 MachineFunction &MF = *MBB->getParent(); 2142 MachineRegisterInfo &MRI = MF.getRegInfo(); 2143 2144 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2145 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LA), Reg) 2146 .addOperand(Base).addImm(0).addReg(0); 2147 return Reg; 2148} 2149 2150// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 2151MachineBasicBlock * 2152SystemZTargetLowering::emitSelect(MachineInstr *MI, 2153 MachineBasicBlock *MBB) const { 2154 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2155 2156 unsigned DestReg = MI->getOperand(0).getReg(); 2157 unsigned TrueReg = MI->getOperand(1).getReg(); 2158 unsigned FalseReg = MI->getOperand(2).getReg(); 2159 unsigned CCValid = MI->getOperand(3).getImm(); 2160 unsigned CCMask = MI->getOperand(4).getImm(); 2161 DebugLoc DL = MI->getDebugLoc(); 2162 2163 MachineBasicBlock *StartMBB = MBB; 2164 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2165 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2166 2167 // StartMBB: 2168 // BRC CCMask, JoinMBB 2169 // # fallthrough to FalseMBB 2170 MBB = StartMBB; 2171 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2172 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2173 MBB->addSuccessor(JoinMBB); 2174 MBB->addSuccessor(FalseMBB); 2175 2176 // FalseMBB: 2177 // # fallthrough to JoinMBB 2178 MBB = FalseMBB; 2179 MBB->addSuccessor(JoinMBB); 2180 2181 // JoinMBB: 2182 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 2183 // ... 2184 MBB = JoinMBB; 2185 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg) 2186 .addReg(TrueReg).addMBB(StartMBB) 2187 .addReg(FalseReg).addMBB(FalseMBB); 2188 2189 MI->eraseFromParent(); 2190 return JoinMBB; 2191} 2192 2193// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 2194// StoreOpcode is the store to use and Invert says whether the store should 2195// happen when the condition is false rather than true. If a STORE ON 2196// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 2197MachineBasicBlock * 2198SystemZTargetLowering::emitCondStore(MachineInstr *MI, 2199 MachineBasicBlock *MBB, 2200 unsigned StoreOpcode, unsigned STOCOpcode, 2201 bool Invert) const { 2202 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2203 2204 unsigned SrcReg = MI->getOperand(0).getReg(); 2205 MachineOperand Base = MI->getOperand(1); 2206 int64_t Disp = MI->getOperand(2).getImm(); 2207 unsigned IndexReg = MI->getOperand(3).getReg(); 2208 unsigned CCValid = MI->getOperand(4).getImm(); 2209 unsigned CCMask = MI->getOperand(5).getImm(); 2210 DebugLoc DL = MI->getDebugLoc(); 2211 2212 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 2213 2214 // Use STOCOpcode if possible. We could use different store patterns in 2215 // order to avoid matching the index register, but the performance trade-offs 2216 // might be more complicated in that case. 2217 if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) { 2218 if (Invert) 2219 CCMask ^= CCValid; 2220 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 2221 .addReg(SrcReg).addOperand(Base).addImm(Disp) 2222 .addImm(CCValid).addImm(CCMask); 2223 MI->eraseFromParent(); 2224 return MBB; 2225 } 2226 2227 // Get the condition needed to branch around the store. 2228 if (!Invert) 2229 CCMask ^= CCValid; 2230 2231 MachineBasicBlock *StartMBB = MBB; 2232 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2233 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2234 2235 // StartMBB: 2236 // BRC CCMask, JoinMBB 2237 // # fallthrough to FalseMBB 2238 MBB = StartMBB; 2239 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2240 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2241 MBB->addSuccessor(JoinMBB); 2242 MBB->addSuccessor(FalseMBB); 2243 2244 // FalseMBB: 2245 // store %SrcReg, %Disp(%Index,%Base) 2246 // # fallthrough to JoinMBB 2247 MBB = FalseMBB; 2248 BuildMI(MBB, DL, TII->get(StoreOpcode)) 2249 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 2250 MBB->addSuccessor(JoinMBB); 2251 2252 MI->eraseFromParent(); 2253 return JoinMBB; 2254} 2255 2256// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 2257// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 2258// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 2259// BitSize is the width of the field in bits, or 0 if this is a partword 2260// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 2261// is one of the operands. Invert says whether the field should be 2262// inverted after performing BinOpcode (e.g. for NAND). 2263MachineBasicBlock * 2264SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 2265 MachineBasicBlock *MBB, 2266 unsigned BinOpcode, 2267 unsigned BitSize, 2268 bool Invert) const { 2269 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2270 MachineFunction &MF = *MBB->getParent(); 2271 MachineRegisterInfo &MRI = MF.getRegInfo(); 2272 bool IsSubWord = (BitSize < 32); 2273 2274 // Extract the operands. Base can be a register or a frame index. 2275 // Src2 can be a register or immediate. 2276 unsigned Dest = MI->getOperand(0).getReg(); 2277 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2278 int64_t Disp = MI->getOperand(2).getImm(); 2279 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2280 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2281 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2282 DebugLoc DL = MI->getDebugLoc(); 2283 if (IsSubWord) 2284 BitSize = MI->getOperand(6).getImm(); 2285 2286 // Subword operations use 32-bit registers. 2287 const TargetRegisterClass *RC = (BitSize <= 32 ? 2288 &SystemZ::GR32BitRegClass : 2289 &SystemZ::GR64BitRegClass); 2290 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2291 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2292 2293 // Get the right opcodes for the displacement. 2294 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2295 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2296 assert(LOpcode && CSOpcode && "Displacement out of range"); 2297 2298 // Create virtual registers for temporary results. 2299 unsigned OrigVal = MRI.createVirtualRegister(RC); 2300 unsigned OldVal = MRI.createVirtualRegister(RC); 2301 unsigned NewVal = (BinOpcode || IsSubWord ? 2302 MRI.createVirtualRegister(RC) : Src2.getReg()); 2303 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2304 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2305 2306 // Insert a basic block for the main loop. 2307 MachineBasicBlock *StartMBB = MBB; 2308 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2309 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2310 2311 // StartMBB: 2312 // ... 2313 // %OrigVal = L Disp(%Base) 2314 // # fall through to LoopMMB 2315 MBB = StartMBB; 2316 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2317 .addOperand(Base).addImm(Disp).addReg(0); 2318 MBB->addSuccessor(LoopMBB); 2319 2320 // LoopMBB: 2321 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2322 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2323 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2324 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2325 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2326 // JNE LoopMBB 2327 // # fall through to DoneMMB 2328 MBB = LoopMBB; 2329 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2330 .addReg(OrigVal).addMBB(StartMBB) 2331 .addReg(Dest).addMBB(LoopMBB); 2332 if (IsSubWord) 2333 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2334 .addReg(OldVal).addReg(BitShift).addImm(0); 2335 if (Invert) { 2336 // Perform the operation normally and then invert every bit of the field. 2337 unsigned Tmp = MRI.createVirtualRegister(RC); 2338 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2339 .addReg(RotatedOldVal).addOperand(Src2); 2340 if (BitSize < 32) 2341 // XILF with the upper BitSize bits set. 2342 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2343 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 2344 else if (BitSize == 32) 2345 // XILF with every bit set. 2346 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2347 .addReg(Tmp).addImm(~uint32_t(0)); 2348 else { 2349 // Use LCGR and add -1 to the result, which is more compact than 2350 // an XILF, XILH pair. 2351 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2352 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2353 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2354 .addReg(Tmp2).addImm(-1); 2355 } 2356 } else if (BinOpcode) 2357 // A simply binary operation. 2358 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2359 .addReg(RotatedOldVal).addOperand(Src2); 2360 else if (IsSubWord) 2361 // Use RISBG to rotate Src2 into position and use it to replace the 2362 // field in RotatedOldVal. 2363 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2364 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2365 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2366 if (IsSubWord) 2367 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2368 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2369 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2370 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2371 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2372 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2373 MBB->addSuccessor(LoopMBB); 2374 MBB->addSuccessor(DoneMBB); 2375 2376 MI->eraseFromParent(); 2377 return DoneMBB; 2378} 2379 2380// Implement EmitInstrWithCustomInserter for pseudo 2381// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2382// instruction that should be used to compare the current field with the 2383// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2384// for when the current field should be kept. BitSize is the width of 2385// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2386MachineBasicBlock * 2387SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2388 MachineBasicBlock *MBB, 2389 unsigned CompareOpcode, 2390 unsigned KeepOldMask, 2391 unsigned BitSize) const { 2392 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2393 MachineFunction &MF = *MBB->getParent(); 2394 MachineRegisterInfo &MRI = MF.getRegInfo(); 2395 bool IsSubWord = (BitSize < 32); 2396 2397 // Extract the operands. Base can be a register or a frame index. 2398 unsigned Dest = MI->getOperand(0).getReg(); 2399 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2400 int64_t Disp = MI->getOperand(2).getImm(); 2401 unsigned Src2 = MI->getOperand(3).getReg(); 2402 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2403 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2404 DebugLoc DL = MI->getDebugLoc(); 2405 if (IsSubWord) 2406 BitSize = MI->getOperand(6).getImm(); 2407 2408 // Subword operations use 32-bit registers. 2409 const TargetRegisterClass *RC = (BitSize <= 32 ? 2410 &SystemZ::GR32BitRegClass : 2411 &SystemZ::GR64BitRegClass); 2412 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2413 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2414 2415 // Get the right opcodes for the displacement. 2416 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2417 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2418 assert(LOpcode && CSOpcode && "Displacement out of range"); 2419 2420 // Create virtual registers for temporary results. 2421 unsigned OrigVal = MRI.createVirtualRegister(RC); 2422 unsigned OldVal = MRI.createVirtualRegister(RC); 2423 unsigned NewVal = MRI.createVirtualRegister(RC); 2424 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2425 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2426 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2427 2428 // Insert 3 basic blocks for the loop. 2429 MachineBasicBlock *StartMBB = MBB; 2430 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2431 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2432 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2433 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2434 2435 // StartMBB: 2436 // ... 2437 // %OrigVal = L Disp(%Base) 2438 // # fall through to LoopMMB 2439 MBB = StartMBB; 2440 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2441 .addOperand(Base).addImm(Disp).addReg(0); 2442 MBB->addSuccessor(LoopMBB); 2443 2444 // LoopMBB: 2445 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2446 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2447 // CompareOpcode %RotatedOldVal, %Src2 2448 // BRC KeepOldMask, UpdateMBB 2449 MBB = LoopMBB; 2450 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2451 .addReg(OrigVal).addMBB(StartMBB) 2452 .addReg(Dest).addMBB(UpdateMBB); 2453 if (IsSubWord) 2454 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2455 .addReg(OldVal).addReg(BitShift).addImm(0); 2456 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2457 .addReg(RotatedOldVal).addReg(Src2); 2458 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2459 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2460 MBB->addSuccessor(UpdateMBB); 2461 MBB->addSuccessor(UseAltMBB); 2462 2463 // UseAltMBB: 2464 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2465 // # fall through to UpdateMMB 2466 MBB = UseAltMBB; 2467 if (IsSubWord) 2468 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2469 .addReg(RotatedOldVal).addReg(Src2) 2470 .addImm(32).addImm(31 + BitSize).addImm(0); 2471 MBB->addSuccessor(UpdateMBB); 2472 2473 // UpdateMBB: 2474 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2475 // [ %RotatedAltVal, UseAltMBB ] 2476 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2477 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2478 // JNE LoopMBB 2479 // # fall through to DoneMMB 2480 MBB = UpdateMBB; 2481 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2482 .addReg(RotatedOldVal).addMBB(LoopMBB) 2483 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2484 if (IsSubWord) 2485 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2486 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2487 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2488 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2489 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2490 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2491 MBB->addSuccessor(LoopMBB); 2492 MBB->addSuccessor(DoneMBB); 2493 2494 MI->eraseFromParent(); 2495 return DoneMBB; 2496} 2497 2498// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2499// instruction MI. 2500MachineBasicBlock * 2501SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2502 MachineBasicBlock *MBB) const { 2503 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2504 MachineFunction &MF = *MBB->getParent(); 2505 MachineRegisterInfo &MRI = MF.getRegInfo(); 2506 2507 // Extract the operands. Base can be a register or a frame index. 2508 unsigned Dest = MI->getOperand(0).getReg(); 2509 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2510 int64_t Disp = MI->getOperand(2).getImm(); 2511 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2512 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2513 unsigned BitShift = MI->getOperand(5).getReg(); 2514 unsigned NegBitShift = MI->getOperand(6).getReg(); 2515 int64_t BitSize = MI->getOperand(7).getImm(); 2516 DebugLoc DL = MI->getDebugLoc(); 2517 2518 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2519 2520 // Get the right opcodes for the displacement. 2521 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2522 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2523 assert(LOpcode && CSOpcode && "Displacement out of range"); 2524 2525 // Create virtual registers for temporary results. 2526 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2527 unsigned OldVal = MRI.createVirtualRegister(RC); 2528 unsigned CmpVal = MRI.createVirtualRegister(RC); 2529 unsigned SwapVal = MRI.createVirtualRegister(RC); 2530 unsigned StoreVal = MRI.createVirtualRegister(RC); 2531 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2532 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2533 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2534 2535 // Insert 2 basic blocks for the loop. 2536 MachineBasicBlock *StartMBB = MBB; 2537 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2538 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2539 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 2540 2541 // StartMBB: 2542 // ... 2543 // %OrigOldVal = L Disp(%Base) 2544 // # fall through to LoopMMB 2545 MBB = StartMBB; 2546 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 2547 .addOperand(Base).addImm(Disp).addReg(0); 2548 MBB->addSuccessor(LoopMBB); 2549 2550 // LoopMBB: 2551 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 2552 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 2553 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 2554 // %Dest = RLL %OldVal, BitSize(%BitShift) 2555 // ^^ The low BitSize bits contain the field 2556 // of interest. 2557 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 2558 // ^^ Replace the upper 32-BitSize bits of the 2559 // comparison value with those that we loaded, 2560 // so that we can use a full word comparison. 2561 // CR %Dest, %RetryCmpVal 2562 // JNE DoneMBB 2563 // # Fall through to SetMBB 2564 MBB = LoopMBB; 2565 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2566 .addReg(OrigOldVal).addMBB(StartMBB) 2567 .addReg(RetryOldVal).addMBB(SetMBB); 2568 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 2569 .addReg(OrigCmpVal).addMBB(StartMBB) 2570 .addReg(RetryCmpVal).addMBB(SetMBB); 2571 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 2572 .addReg(OrigSwapVal).addMBB(StartMBB) 2573 .addReg(RetrySwapVal).addMBB(SetMBB); 2574 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 2575 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 2576 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 2577 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2578 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 2579 .addReg(Dest).addReg(RetryCmpVal); 2580 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2581 .addImm(SystemZ::CCMASK_ICMP) 2582 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 2583 MBB->addSuccessor(DoneMBB); 2584 MBB->addSuccessor(SetMBB); 2585 2586 // SetMBB: 2587 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 2588 // ^^ Replace the upper 32-BitSize bits of the new 2589 // value with those that we loaded. 2590 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 2591 // ^^ Rotate the new field to its proper position. 2592 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 2593 // JNE LoopMBB 2594 // # fall through to ExitMMB 2595 MBB = SetMBB; 2596 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 2597 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2598 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 2599 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 2600 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2601 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2602 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2603 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2604 MBB->addSuccessor(LoopMBB); 2605 MBB->addSuccessor(DoneMBB); 2606 2607 MI->eraseFromParent(); 2608 return DoneMBB; 2609} 2610 2611// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2612// if the high register of the GR128 value must be cleared or false if 2613// it's "don't care". SubReg is subreg_l32 when extending a GR32 2614// and subreg_l64 when extending a GR64. 2615MachineBasicBlock * 2616SystemZTargetLowering::emitExt128(MachineInstr *MI, 2617 MachineBasicBlock *MBB, 2618 bool ClearEven, unsigned SubReg) const { 2619 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2620 MachineFunction &MF = *MBB->getParent(); 2621 MachineRegisterInfo &MRI = MF.getRegInfo(); 2622 DebugLoc DL = MI->getDebugLoc(); 2623 2624 unsigned Dest = MI->getOperand(0).getReg(); 2625 unsigned Src = MI->getOperand(1).getReg(); 2626 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2627 2628 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2629 if (ClearEven) { 2630 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2631 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2632 2633 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2634 .addImm(0); 2635 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2636 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64); 2637 In128 = NewIn128; 2638 } 2639 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2640 .addReg(In128).addReg(Src).addImm(SubReg); 2641 2642 MI->eraseFromParent(); 2643 return MBB; 2644} 2645 2646MachineBasicBlock * 2647SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 2648 MachineBasicBlock *MBB, 2649 unsigned Opcode) const { 2650 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2651 MachineFunction &MF = *MBB->getParent(); 2652 MachineRegisterInfo &MRI = MF.getRegInfo(); 2653 DebugLoc DL = MI->getDebugLoc(); 2654 2655 MachineOperand DestBase = earlyUseOperand(MI->getOperand(0)); 2656 uint64_t DestDisp = MI->getOperand(1).getImm(); 2657 MachineOperand SrcBase = earlyUseOperand(MI->getOperand(2)); 2658 uint64_t SrcDisp = MI->getOperand(3).getImm(); 2659 uint64_t Length = MI->getOperand(4).getImm(); 2660 2661 // When generating more than one CLC, all but the last will need to 2662 // branch to the end when a difference is found. 2663 MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ? 2664 splitBlockAfter(MI, MBB) : 0); 2665 2666 // Check for the loop form, in which operand 5 is the trip count. 2667 if (MI->getNumExplicitOperands() > 5) { 2668 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); 2669 2670 uint64_t StartCountReg = MI->getOperand(5).getReg(); 2671 uint64_t StartSrcReg = forceReg(MI, SrcBase, TII); 2672 uint64_t StartDestReg = (HaveSingleBase ? StartSrcReg : 2673 forceReg(MI, DestBase, TII)); 2674 2675 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass; 2676 uint64_t ThisSrcReg = MRI.createVirtualRegister(RC); 2677 uint64_t ThisDestReg = (HaveSingleBase ? ThisSrcReg : 2678 MRI.createVirtualRegister(RC)); 2679 uint64_t NextSrcReg = MRI.createVirtualRegister(RC); 2680 uint64_t NextDestReg = (HaveSingleBase ? NextSrcReg : 2681 MRI.createVirtualRegister(RC)); 2682 2683 RC = &SystemZ::GR64BitRegClass; 2684 uint64_t ThisCountReg = MRI.createVirtualRegister(RC); 2685 uint64_t NextCountReg = MRI.createVirtualRegister(RC); 2686 2687 MachineBasicBlock *StartMBB = MBB; 2688 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2689 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2690 MachineBasicBlock *NextMBB = (EndMBB ? emitBlockAfter(LoopMBB) : LoopMBB); 2691 2692 // StartMBB: 2693 // # fall through to LoopMMB 2694 MBB->addSuccessor(LoopMBB); 2695 2696 // LoopMBB: 2697 // %ThisDestReg = phi [ %StartDestReg, StartMBB ], 2698 // [ %NextDestReg, NextMBB ] 2699 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ], 2700 // [ %NextSrcReg, NextMBB ] 2701 // %ThisCountReg = phi [ %StartCountReg, StartMBB ], 2702 // [ %NextCountReg, NextMBB ] 2703 // ( PFD 2, 768+DestDisp(%ThisDestReg) ) 2704 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg) 2705 // ( JLH EndMBB ) 2706 // 2707 // The prefetch is used only for MVC. The JLH is used only for CLC. 2708 MBB = LoopMBB; 2709 2710 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) 2711 .addReg(StartDestReg).addMBB(StartMBB) 2712 .addReg(NextDestReg).addMBB(NextMBB); 2713 if (!HaveSingleBase) 2714 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) 2715 .addReg(StartSrcReg).addMBB(StartMBB) 2716 .addReg(NextSrcReg).addMBB(NextMBB); 2717 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) 2718 .addReg(StartCountReg).addMBB(StartMBB) 2719 .addReg(NextCountReg).addMBB(NextMBB); 2720 if (Opcode == SystemZ::MVC) 2721 BuildMI(MBB, DL, TII->get(SystemZ::PFD)) 2722 .addImm(SystemZ::PFD_WRITE) 2723 .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); 2724 BuildMI(MBB, DL, TII->get(Opcode)) 2725 .addReg(ThisDestReg).addImm(DestDisp).addImm(256) 2726 .addReg(ThisSrcReg).addImm(SrcDisp); 2727 if (EndMBB) { 2728 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2729 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2730 .addMBB(EndMBB); 2731 MBB->addSuccessor(EndMBB); 2732 MBB->addSuccessor(NextMBB); 2733 } 2734 2735 // NextMBB: 2736 // %NextDestReg = LA 256(%ThisDestReg) 2737 // %NextSrcReg = LA 256(%ThisSrcReg) 2738 // %NextCountReg = AGHI %ThisCountReg, -1 2739 // CGHI %NextCountReg, 0 2740 // JLH LoopMBB 2741 // # fall through to DoneMMB 2742 // 2743 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes. 2744 MBB = NextMBB; 2745 2746 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) 2747 .addReg(ThisDestReg).addImm(256).addReg(0); 2748 if (!HaveSingleBase) 2749 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) 2750 .addReg(ThisSrcReg).addImm(256).addReg(0); 2751 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) 2752 .addReg(ThisCountReg).addImm(-1); 2753 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) 2754 .addReg(NextCountReg).addImm(0); 2755 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2756 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2757 .addMBB(LoopMBB); 2758 MBB->addSuccessor(LoopMBB); 2759 MBB->addSuccessor(DoneMBB); 2760 2761 DestBase = MachineOperand::CreateReg(NextDestReg, false); 2762 SrcBase = MachineOperand::CreateReg(NextSrcReg, false); 2763 Length &= 255; 2764 MBB = DoneMBB; 2765 } 2766 // Handle any remaining bytes with straight-line code. 2767 while (Length > 0) { 2768 uint64_t ThisLength = std::min(Length, uint64_t(256)); 2769 // The previous iteration might have created out-of-range displacements. 2770 // Apply them using LAY if so. 2771 if (!isUInt<12>(DestDisp)) { 2772 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2773 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 2774 .addOperand(DestBase).addImm(DestDisp).addReg(0); 2775 DestBase = MachineOperand::CreateReg(Reg, false); 2776 DestDisp = 0; 2777 } 2778 if (!isUInt<12>(SrcDisp)) { 2779 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2780 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 2781 .addOperand(SrcBase).addImm(SrcDisp).addReg(0); 2782 SrcBase = MachineOperand::CreateReg(Reg, false); 2783 SrcDisp = 0; 2784 } 2785 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 2786 .addOperand(DestBase).addImm(DestDisp).addImm(ThisLength) 2787 .addOperand(SrcBase).addImm(SrcDisp); 2788 DestDisp += ThisLength; 2789 SrcDisp += ThisLength; 2790 Length -= ThisLength; 2791 // If there's another CLC to go, branch to the end if a difference 2792 // was found. 2793 if (EndMBB && Length > 0) { 2794 MachineBasicBlock *NextMBB = splitBlockBefore(MI, MBB); 2795 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2796 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2797 .addMBB(EndMBB); 2798 MBB->addSuccessor(EndMBB); 2799 MBB->addSuccessor(NextMBB); 2800 MBB = NextMBB; 2801 } 2802 } 2803 if (EndMBB) { 2804 MBB->addSuccessor(EndMBB); 2805 MBB = EndMBB; 2806 MBB->addLiveIn(SystemZ::CC); 2807 } 2808 2809 MI->eraseFromParent(); 2810 return MBB; 2811} 2812 2813// Decompose string pseudo-instruction MI into a loop that continually performs 2814// Opcode until CC != 3. 2815MachineBasicBlock * 2816SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 2817 MachineBasicBlock *MBB, 2818 unsigned Opcode) const { 2819 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2820 MachineFunction &MF = *MBB->getParent(); 2821 MachineRegisterInfo &MRI = MF.getRegInfo(); 2822 DebugLoc DL = MI->getDebugLoc(); 2823 2824 uint64_t End1Reg = MI->getOperand(0).getReg(); 2825 uint64_t Start1Reg = MI->getOperand(1).getReg(); 2826 uint64_t Start2Reg = MI->getOperand(2).getReg(); 2827 uint64_t CharReg = MI->getOperand(3).getReg(); 2828 2829 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 2830 uint64_t This1Reg = MRI.createVirtualRegister(RC); 2831 uint64_t This2Reg = MRI.createVirtualRegister(RC); 2832 uint64_t End2Reg = MRI.createVirtualRegister(RC); 2833 2834 MachineBasicBlock *StartMBB = MBB; 2835 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2836 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2837 2838 // StartMBB: 2839 // # fall through to LoopMMB 2840 MBB->addSuccessor(LoopMBB); 2841 2842 // LoopMBB: 2843 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 2844 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 2845 // R0L = %CharReg 2846 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L 2847 // JO LoopMBB 2848 // # fall through to DoneMMB 2849 // 2850 // The load of R0L can be hoisted by post-RA LICM. 2851 MBB = LoopMBB; 2852 2853 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 2854 .addReg(Start1Reg).addMBB(StartMBB) 2855 .addReg(End1Reg).addMBB(LoopMBB); 2856 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 2857 .addReg(Start2Reg).addMBB(StartMBB) 2858 .addReg(End2Reg).addMBB(LoopMBB); 2859 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); 2860 BuildMI(MBB, DL, TII->get(Opcode)) 2861 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 2862 .addReg(This1Reg).addReg(This2Reg); 2863 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2864 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 2865 MBB->addSuccessor(LoopMBB); 2866 MBB->addSuccessor(DoneMBB); 2867 2868 DoneMBB->addLiveIn(SystemZ::CC); 2869 2870 MI->eraseFromParent(); 2871 return DoneMBB; 2872} 2873 2874MachineBasicBlock *SystemZTargetLowering:: 2875EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 2876 switch (MI->getOpcode()) { 2877 case SystemZ::Select32Mux: 2878 case SystemZ::Select32: 2879 case SystemZ::SelectF32: 2880 case SystemZ::Select64: 2881 case SystemZ::SelectF64: 2882 case SystemZ::SelectF128: 2883 return emitSelect(MI, MBB); 2884 2885 case SystemZ::CondStore8: 2886 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 2887 case SystemZ::CondStore8Inv: 2888 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 2889 case SystemZ::CondStore16: 2890 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 2891 case SystemZ::CondStore16Inv: 2892 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 2893 case SystemZ::CondStore32: 2894 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 2895 case SystemZ::CondStore32Inv: 2896 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 2897 case SystemZ::CondStore64: 2898 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 2899 case SystemZ::CondStore64Inv: 2900 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 2901 case SystemZ::CondStoreF32: 2902 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 2903 case SystemZ::CondStoreF32Inv: 2904 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 2905 case SystemZ::CondStoreF64: 2906 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 2907 case SystemZ::CondStoreF64Inv: 2908 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 2909 2910 case SystemZ::AEXT128_64: 2911 return emitExt128(MI, MBB, false, SystemZ::subreg_l64); 2912 case SystemZ::ZEXT128_32: 2913 return emitExt128(MI, MBB, true, SystemZ::subreg_l32); 2914 case SystemZ::ZEXT128_64: 2915 return emitExt128(MI, MBB, true, SystemZ::subreg_l64); 2916 2917 case SystemZ::ATOMIC_SWAPW: 2918 return emitAtomicLoadBinary(MI, MBB, 0, 0); 2919 case SystemZ::ATOMIC_SWAP_32: 2920 return emitAtomicLoadBinary(MI, MBB, 0, 32); 2921 case SystemZ::ATOMIC_SWAP_64: 2922 return emitAtomicLoadBinary(MI, MBB, 0, 64); 2923 2924 case SystemZ::ATOMIC_LOADW_AR: 2925 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 2926 case SystemZ::ATOMIC_LOADW_AFI: 2927 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 2928 case SystemZ::ATOMIC_LOAD_AR: 2929 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 2930 case SystemZ::ATOMIC_LOAD_AHI: 2931 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 2932 case SystemZ::ATOMIC_LOAD_AFI: 2933 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 2934 case SystemZ::ATOMIC_LOAD_AGR: 2935 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 2936 case SystemZ::ATOMIC_LOAD_AGHI: 2937 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 2938 case SystemZ::ATOMIC_LOAD_AGFI: 2939 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 2940 2941 case SystemZ::ATOMIC_LOADW_SR: 2942 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 2943 case SystemZ::ATOMIC_LOAD_SR: 2944 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 2945 case SystemZ::ATOMIC_LOAD_SGR: 2946 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 2947 2948 case SystemZ::ATOMIC_LOADW_NR: 2949 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 2950 case SystemZ::ATOMIC_LOADW_NILH: 2951 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); 2952 case SystemZ::ATOMIC_LOAD_NR: 2953 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 2954 case SystemZ::ATOMIC_LOAD_NILL: 2955 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); 2956 case SystemZ::ATOMIC_LOAD_NILH: 2957 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); 2958 case SystemZ::ATOMIC_LOAD_NILF: 2959 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); 2960 case SystemZ::ATOMIC_LOAD_NGR: 2961 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 2962 case SystemZ::ATOMIC_LOAD_NILL64: 2963 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); 2964 case SystemZ::ATOMIC_LOAD_NILH64: 2965 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); 2966 case SystemZ::ATOMIC_LOAD_NIHL: 2967 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); 2968 case SystemZ::ATOMIC_LOAD_NIHH: 2969 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); 2970 case SystemZ::ATOMIC_LOAD_NILF64: 2971 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); 2972 case SystemZ::ATOMIC_LOAD_NIHF: 2973 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); 2974 2975 case SystemZ::ATOMIC_LOADW_OR: 2976 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 2977 case SystemZ::ATOMIC_LOADW_OILH: 2978 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); 2979 case SystemZ::ATOMIC_LOAD_OR: 2980 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 2981 case SystemZ::ATOMIC_LOAD_OILL: 2982 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); 2983 case SystemZ::ATOMIC_LOAD_OILH: 2984 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); 2985 case SystemZ::ATOMIC_LOAD_OILF: 2986 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); 2987 case SystemZ::ATOMIC_LOAD_OGR: 2988 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 2989 case SystemZ::ATOMIC_LOAD_OILL64: 2990 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); 2991 case SystemZ::ATOMIC_LOAD_OILH64: 2992 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); 2993 case SystemZ::ATOMIC_LOAD_OIHL64: 2994 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); 2995 case SystemZ::ATOMIC_LOAD_OIHH64: 2996 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); 2997 case SystemZ::ATOMIC_LOAD_OILF64: 2998 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); 2999 case SystemZ::ATOMIC_LOAD_OIHF64: 3000 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); 3001 3002 case SystemZ::ATOMIC_LOADW_XR: 3003 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 3004 case SystemZ::ATOMIC_LOADW_XILF: 3005 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); 3006 case SystemZ::ATOMIC_LOAD_XR: 3007 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 3008 case SystemZ::ATOMIC_LOAD_XILF: 3009 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); 3010 case SystemZ::ATOMIC_LOAD_XGR: 3011 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 3012 case SystemZ::ATOMIC_LOAD_XILF64: 3013 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); 3014 case SystemZ::ATOMIC_LOAD_XIHF: 3015 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); 3016 3017 case SystemZ::ATOMIC_LOADW_NRi: 3018 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 3019 case SystemZ::ATOMIC_LOADW_NILHi: 3020 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); 3021 case SystemZ::ATOMIC_LOAD_NRi: 3022 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 3023 case SystemZ::ATOMIC_LOAD_NILLi: 3024 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); 3025 case SystemZ::ATOMIC_LOAD_NILHi: 3026 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); 3027 case SystemZ::ATOMIC_LOAD_NILFi: 3028 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); 3029 case SystemZ::ATOMIC_LOAD_NGRi: 3030 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 3031 case SystemZ::ATOMIC_LOAD_NILL64i: 3032 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); 3033 case SystemZ::ATOMIC_LOAD_NILH64i: 3034 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); 3035 case SystemZ::ATOMIC_LOAD_NIHLi: 3036 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); 3037 case SystemZ::ATOMIC_LOAD_NIHHi: 3038 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); 3039 case SystemZ::ATOMIC_LOAD_NILF64i: 3040 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); 3041 case SystemZ::ATOMIC_LOAD_NIHFi: 3042 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); 3043 3044 case SystemZ::ATOMIC_LOADW_MIN: 3045 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3046 SystemZ::CCMASK_CMP_LE, 0); 3047 case SystemZ::ATOMIC_LOAD_MIN_32: 3048 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3049 SystemZ::CCMASK_CMP_LE, 32); 3050 case SystemZ::ATOMIC_LOAD_MIN_64: 3051 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3052 SystemZ::CCMASK_CMP_LE, 64); 3053 3054 case SystemZ::ATOMIC_LOADW_MAX: 3055 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3056 SystemZ::CCMASK_CMP_GE, 0); 3057 case SystemZ::ATOMIC_LOAD_MAX_32: 3058 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3059 SystemZ::CCMASK_CMP_GE, 32); 3060 case SystemZ::ATOMIC_LOAD_MAX_64: 3061 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3062 SystemZ::CCMASK_CMP_GE, 64); 3063 3064 case SystemZ::ATOMIC_LOADW_UMIN: 3065 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3066 SystemZ::CCMASK_CMP_LE, 0); 3067 case SystemZ::ATOMIC_LOAD_UMIN_32: 3068 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3069 SystemZ::CCMASK_CMP_LE, 32); 3070 case SystemZ::ATOMIC_LOAD_UMIN_64: 3071 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3072 SystemZ::CCMASK_CMP_LE, 64); 3073 3074 case SystemZ::ATOMIC_LOADW_UMAX: 3075 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3076 SystemZ::CCMASK_CMP_GE, 0); 3077 case SystemZ::ATOMIC_LOAD_UMAX_32: 3078 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3079 SystemZ::CCMASK_CMP_GE, 32); 3080 case SystemZ::ATOMIC_LOAD_UMAX_64: 3081 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3082 SystemZ::CCMASK_CMP_GE, 64); 3083 3084 case SystemZ::ATOMIC_CMP_SWAPW: 3085 return emitAtomicCmpSwapW(MI, MBB); 3086 case SystemZ::MVCSequence: 3087 case SystemZ::MVCLoop: 3088 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 3089 case SystemZ::NCSequence: 3090 case SystemZ::NCLoop: 3091 return emitMemMemWrapper(MI, MBB, SystemZ::NC); 3092 case SystemZ::OCSequence: 3093 case SystemZ::OCLoop: 3094 return emitMemMemWrapper(MI, MBB, SystemZ::OC); 3095 case SystemZ::XCSequence: 3096 case SystemZ::XCLoop: 3097 return emitMemMemWrapper(MI, MBB, SystemZ::XC); 3098 case SystemZ::CLCSequence: 3099 case SystemZ::CLCLoop: 3100 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 3101 case SystemZ::CLSTLoop: 3102 return emitStringWrapper(MI, MBB, SystemZ::CLST); 3103 case SystemZ::MVSTLoop: 3104 return emitStringWrapper(MI, MBB, SystemZ::MVST); 3105 case SystemZ::SRSTLoop: 3106 return emitStringWrapper(MI, MBB, SystemZ::SRST); 3107 default: 3108 llvm_unreachable("Unexpected instr type to insert"); 3109 } 3110} 3111