SystemZISelLowering.cpp revision d50bcb2162a529534da42748ab4a418bfc9aaf06
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "systemz-lower" 15 16#include "SystemZISelLowering.h" 17#include "SystemZCallingConv.h" 18#include "SystemZConstantPoolValue.h" 19#include "SystemZMachineFunctionInfo.h" 20#include "SystemZTargetMachine.h" 21#include "llvm/CodeGen/CallingConvLower.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25 26using namespace llvm; 27 28// Classify VT as either 32 or 64 bit. 29static bool is32Bit(EVT VT) { 30 switch (VT.getSimpleVT().SimpleTy) { 31 case MVT::i32: 32 return true; 33 case MVT::i64: 34 return false; 35 default: 36 llvm_unreachable("Unsupported type"); 37 } 38} 39 40// Return a version of MachineOperand that can be safely used before the 41// final use. 42static MachineOperand earlyUseOperand(MachineOperand Op) { 43 if (Op.isReg()) 44 Op.setIsKill(false); 45 return Op; 46} 47 48SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 49 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 50 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 51 MVT PtrVT = getPointerTy(); 52 53 // Set up the register classes. 54 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 55 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 56 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 57 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 58 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 59 60 // Compute derived properties from the register classes 61 computeRegisterProperties(); 62 63 // Set up special registers. 64 setExceptionPointerRegister(SystemZ::R6D); 65 setExceptionSelectorRegister(SystemZ::R7D); 66 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 67 68 // TODO: It may be better to default to latency-oriented scheduling, however 69 // LLVM's current latency-oriented scheduler can't handle physreg definitions 70 // such as SystemZ has with CC, so set this to the register-pressure 71 // scheduler, because it can. 72 setSchedulingPreference(Sched::RegPressure); 73 74 setBooleanContents(ZeroOrOneBooleanContent); 75 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 76 77 // Instructions are strings of 2-byte aligned 2-byte values. 78 setMinFunctionAlignment(2); 79 80 // Handle operations that are handled in a similar way for all types. 81 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 82 I <= MVT::LAST_FP_VALUETYPE; 83 ++I) { 84 MVT VT = MVT::SimpleValueType(I); 85 if (isTypeLegal(VT)) { 86 // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). 87 setOperationAction(ISD::SETCC, VT, Expand); 88 89 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 90 setOperationAction(ISD::SELECT, VT, Expand); 91 92 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 93 setOperationAction(ISD::SELECT_CC, VT, Custom); 94 setOperationAction(ISD::BR_CC, VT, Custom); 95 } 96 } 97 98 // Expand jump table branches as address arithmetic followed by an 99 // indirect jump. 100 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 101 102 // Expand BRCOND into a BR_CC (see above). 103 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 104 105 // Handle integer types. 106 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 107 I <= MVT::LAST_INTEGER_VALUETYPE; 108 ++I) { 109 MVT VT = MVT::SimpleValueType(I); 110 if (isTypeLegal(VT)) { 111 // Expand individual DIV and REMs into DIVREMs. 112 setOperationAction(ISD::SDIV, VT, Expand); 113 setOperationAction(ISD::UDIV, VT, Expand); 114 setOperationAction(ISD::SREM, VT, Expand); 115 setOperationAction(ISD::UREM, VT, Expand); 116 setOperationAction(ISD::SDIVREM, VT, Custom); 117 setOperationAction(ISD::UDIVREM, VT, Custom); 118 119 // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 120 // FIXME: probably much too conservative. 121 setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 122 setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 123 124 // No special instructions for these. 125 setOperationAction(ISD::CTPOP, VT, Expand); 126 setOperationAction(ISD::CTTZ, VT, Expand); 127 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 128 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 129 setOperationAction(ISD::ROTR, VT, Expand); 130 131 // Use *MUL_LOHI where possible and a wider multiplication otherwise. 132 setOperationAction(ISD::MULHS, VT, Expand); 133 setOperationAction(ISD::MULHU, VT, Expand); 134 135 // We have instructions for signed but not unsigned FP conversion. 136 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 137 } 138 } 139 140 // Type legalization will convert 8- and 16-bit atomic operations into 141 // forms that operate on i32s (but still keeping the original memory VT). 142 // Lower them into full i32 operations. 143 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 144 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 145 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 146 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 147 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 148 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 149 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 150 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 151 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 152 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 153 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 154 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 155 156 // We have instructions for signed but not unsigned FP conversion. 157 // Handle unsigned 32-bit types as signed 64-bit types. 158 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 159 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 160 161 // We have native support for a 64-bit CTLZ, via FLOGR. 162 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 163 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 164 165 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 166 setOperationAction(ISD::OR, MVT::i64, Custom); 167 168 // The architecture has 32-bit SMUL_LOHI and UMUL_LOHI (MR and MLR), 169 // but they aren't really worth using. There is no 64-bit SMUL_LOHI, 170 // but there is a 64-bit UMUL_LOHI: MLGR. 171 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 172 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 173 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 174 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom); 175 176 // FIXME: Can we support these natively? 177 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 178 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 179 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 180 181 // We have native instructions for i8, i16 and i32 extensions, but not i1. 182 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 183 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 184 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 185 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 186 187 // Handle the various types of symbolic address. 188 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 189 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 190 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 191 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 192 setOperationAction(ISD::JumpTable, PtrVT, Custom); 193 194 // We need to handle dynamic allocations specially because of the 195 // 160-byte area at the bottom of the stack. 196 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 197 198 // Use custom expanders so that we can force the function to use 199 // a frame pointer. 200 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 201 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 202 203 // Expand these using getExceptionSelectorRegister() and 204 // getExceptionPointerRegister(). 205 setOperationAction(ISD::EXCEPTIONADDR, PtrVT, Expand); 206 setOperationAction(ISD::EHSELECTION, PtrVT, Expand); 207 208 // Handle floating-point types. 209 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 210 I <= MVT::LAST_FP_VALUETYPE; 211 ++I) { 212 MVT VT = MVT::SimpleValueType(I); 213 if (isTypeLegal(VT)) { 214 // We can use FI for FRINT. 215 setOperationAction(ISD::FRINT, VT, Legal); 216 217 // No special instructions for these. 218 setOperationAction(ISD::FSIN, VT, Expand); 219 setOperationAction(ISD::FCOS, VT, Expand); 220 setOperationAction(ISD::FREM, VT, Expand); 221 } 222 } 223 224 // We have fused multiply-addition for f32 and f64 but not f128. 225 setOperationAction(ISD::FMA, MVT::f32, Legal); 226 setOperationAction(ISD::FMA, MVT::f64, Legal); 227 setOperationAction(ISD::FMA, MVT::f128, Expand); 228 229 // Needed so that we don't try to implement f128 constant loads using 230 // a load-and-extend of a f80 constant (in cases where the constant 231 // would fit in an f80). 232 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 233 234 // Floating-point truncation and stores need to be done separately. 235 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 236 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 237 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 238 239 // We have 64-bit FPR<->GPR moves, but need special handling for 240 // 32-bit forms. 241 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 242 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 243 244 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 245 // structure, but VAEND is a no-op. 246 setOperationAction(ISD::VASTART, MVT::Other, Custom); 247 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 248 setOperationAction(ISD::VAEND, MVT::Other, Expand); 249} 250 251bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 252 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 253 return Imm.isZero() || Imm.isNegZero(); 254} 255 256//===----------------------------------------------------------------------===// 257// Inline asm support 258//===----------------------------------------------------------------------===// 259 260TargetLowering::ConstraintType 261SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 262 if (Constraint.size() == 1) { 263 switch (Constraint[0]) { 264 case 'a': // Address register 265 case 'd': // Data register (equivalent to 'r') 266 case 'f': // Floating-point register 267 case 'r': // General-purpose register 268 return C_RegisterClass; 269 270 case 'Q': // Memory with base and unsigned 12-bit displacement 271 case 'R': // Likewise, plus an index 272 case 'S': // Memory with base and signed 20-bit displacement 273 case 'T': // Likewise, plus an index 274 case 'm': // Equivalent to 'T'. 275 return C_Memory; 276 277 case 'I': // Unsigned 8-bit constant 278 case 'J': // Unsigned 12-bit constant 279 case 'K': // Signed 16-bit constant 280 case 'L': // Signed 20-bit displacement (on all targets we support) 281 case 'M': // 0x7fffffff 282 return C_Other; 283 284 default: 285 break; 286 } 287 } 288 return TargetLowering::getConstraintType(Constraint); 289} 290 291TargetLowering::ConstraintWeight SystemZTargetLowering:: 292getSingleConstraintMatchWeight(AsmOperandInfo &info, 293 const char *constraint) const { 294 ConstraintWeight weight = CW_Invalid; 295 Value *CallOperandVal = info.CallOperandVal; 296 // If we don't have a value, we can't do a match, 297 // but allow it at the lowest weight. 298 if (CallOperandVal == NULL) 299 return CW_Default; 300 Type *type = CallOperandVal->getType(); 301 // Look at the constraint type. 302 switch (*constraint) { 303 default: 304 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 305 break; 306 307 case 'a': // Address register 308 case 'd': // Data register (equivalent to 'r') 309 case 'r': // General-purpose register 310 if (CallOperandVal->getType()->isIntegerTy()) 311 weight = CW_Register; 312 break; 313 314 case 'f': // Floating-point register 315 if (type->isFloatingPointTy()) 316 weight = CW_Register; 317 break; 318 319 case 'I': // Unsigned 8-bit constant 320 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 321 if (isUInt<8>(C->getZExtValue())) 322 weight = CW_Constant; 323 break; 324 325 case 'J': // Unsigned 12-bit constant 326 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 327 if (isUInt<12>(C->getZExtValue())) 328 weight = CW_Constant; 329 break; 330 331 case 'K': // Signed 16-bit constant 332 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 333 if (isInt<16>(C->getSExtValue())) 334 weight = CW_Constant; 335 break; 336 337 case 'L': // Signed 20-bit displacement (on all targets we support) 338 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 339 if (isInt<20>(C->getSExtValue())) 340 weight = CW_Constant; 341 break; 342 343 case 'M': // 0x7fffffff 344 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 345 if (C->getZExtValue() == 0x7fffffff) 346 weight = CW_Constant; 347 break; 348 } 349 return weight; 350} 351 352std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 353getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const { 354 if (Constraint.size() == 1) { 355 // GCC Constraint Letters 356 switch (Constraint[0]) { 357 default: break; 358 case 'd': // Data register (equivalent to 'r') 359 case 'r': // General-purpose register 360 if (VT == MVT::i64) 361 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 362 else if (VT == MVT::i128) 363 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 364 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 365 366 case 'a': // Address register 367 if (VT == MVT::i64) 368 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 369 else if (VT == MVT::i128) 370 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 371 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 372 373 case 'f': // Floating-point register 374 if (VT == MVT::f64) 375 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 376 else if (VT == MVT::f128) 377 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 378 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 379 } 380 } 381 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 382} 383 384void SystemZTargetLowering:: 385LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 386 std::vector<SDValue> &Ops, 387 SelectionDAG &DAG) const { 388 // Only support length 1 constraints for now. 389 if (Constraint.length() == 1) { 390 switch (Constraint[0]) { 391 case 'I': // Unsigned 8-bit constant 392 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 393 if (isUInt<8>(C->getZExtValue())) 394 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 395 Op.getValueType())); 396 return; 397 398 case 'J': // Unsigned 12-bit constant 399 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 400 if (isUInt<12>(C->getZExtValue())) 401 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 402 Op.getValueType())); 403 return; 404 405 case 'K': // Signed 16-bit constant 406 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 407 if (isInt<16>(C->getSExtValue())) 408 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 409 Op.getValueType())); 410 return; 411 412 case 'L': // Signed 20-bit displacement (on all targets we support) 413 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 414 if (isInt<20>(C->getSExtValue())) 415 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 416 Op.getValueType())); 417 return; 418 419 case 'M': // 0x7fffffff 420 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 421 if (C->getZExtValue() == 0x7fffffff) 422 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 423 Op.getValueType())); 424 return; 425 } 426 } 427 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 428} 429 430//===----------------------------------------------------------------------===// 431// Calling conventions 432//===----------------------------------------------------------------------===// 433 434#include "SystemZGenCallingConv.inc" 435 436// Value is a value that has been passed to us in the location described by VA 437// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 438// any loads onto Chain. 439static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 440 CCValAssign &VA, SDValue Chain, 441 SDValue Value) { 442 // If the argument has been promoted from a smaller type, insert an 443 // assertion to capture this. 444 if (VA.getLocInfo() == CCValAssign::SExt) 445 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 446 DAG.getValueType(VA.getValVT())); 447 else if (VA.getLocInfo() == CCValAssign::ZExt) 448 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 449 DAG.getValueType(VA.getValVT())); 450 451 if (VA.isExtInLoc()) 452 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 453 else if (VA.getLocInfo() == CCValAssign::Indirect) 454 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 455 MachinePointerInfo(), false, false, false, 0); 456 else 457 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 458 return Value; 459} 460 461// Value is a value of type VA.getValVT() that we need to copy into 462// the location described by VA. Return a copy of Value converted to 463// VA.getValVT(). The caller is responsible for handling indirect values. 464static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 465 CCValAssign &VA, SDValue Value) { 466 switch (VA.getLocInfo()) { 467 case CCValAssign::SExt: 468 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 469 case CCValAssign::ZExt: 470 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 471 case CCValAssign::AExt: 472 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 473 case CCValAssign::Full: 474 return Value; 475 default: 476 llvm_unreachable("Unhandled getLocInfo()"); 477 } 478} 479 480SDValue SystemZTargetLowering:: 481LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 482 const SmallVectorImpl<ISD::InputArg> &Ins, 483 SDLoc DL, SelectionDAG &DAG, 484 SmallVectorImpl<SDValue> &InVals) const { 485 MachineFunction &MF = DAG.getMachineFunction(); 486 MachineFrameInfo *MFI = MF.getFrameInfo(); 487 MachineRegisterInfo &MRI = MF.getRegInfo(); 488 SystemZMachineFunctionInfo *FuncInfo = 489 MF.getInfo<SystemZMachineFunctionInfo>(); 490 const SystemZFrameLowering *TFL = 491 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 492 493 // Assign locations to all of the incoming arguments. 494 SmallVector<CCValAssign, 16> ArgLocs; 495 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 496 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 497 498 unsigned NumFixedGPRs = 0; 499 unsigned NumFixedFPRs = 0; 500 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 501 SDValue ArgValue; 502 CCValAssign &VA = ArgLocs[I]; 503 EVT LocVT = VA.getLocVT(); 504 if (VA.isRegLoc()) { 505 // Arguments passed in registers 506 const TargetRegisterClass *RC; 507 switch (LocVT.getSimpleVT().SimpleTy) { 508 default: 509 // Integers smaller than i64 should be promoted to i64. 510 llvm_unreachable("Unexpected argument type"); 511 case MVT::i32: 512 NumFixedGPRs += 1; 513 RC = &SystemZ::GR32BitRegClass; 514 break; 515 case MVT::i64: 516 NumFixedGPRs += 1; 517 RC = &SystemZ::GR64BitRegClass; 518 break; 519 case MVT::f32: 520 NumFixedFPRs += 1; 521 RC = &SystemZ::FP32BitRegClass; 522 break; 523 case MVT::f64: 524 NumFixedFPRs += 1; 525 RC = &SystemZ::FP64BitRegClass; 526 break; 527 } 528 529 unsigned VReg = MRI.createVirtualRegister(RC); 530 MRI.addLiveIn(VA.getLocReg(), VReg); 531 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 532 } else { 533 assert(VA.isMemLoc() && "Argument not register or memory"); 534 535 // Create the frame index object for this incoming parameter. 536 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 537 VA.getLocMemOffset(), true); 538 539 // Create the SelectionDAG nodes corresponding to a load 540 // from this parameter. Unpromoted ints and floats are 541 // passed as right-justified 8-byte values. 542 EVT PtrVT = getPointerTy(); 543 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 544 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 545 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 546 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 547 MachinePointerInfo::getFixedStack(FI), 548 false, false, false, 0); 549 } 550 551 // Convert the value of the argument register into the value that's 552 // being passed. 553 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 554 } 555 556 if (IsVarArg) { 557 // Save the number of non-varargs registers for later use by va_start, etc. 558 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 559 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 560 561 // Likewise the address (in the form of a frame index) of where the 562 // first stack vararg would be. The 1-byte size here is arbitrary. 563 int64_t StackSize = CCInfo.getNextStackOffset(); 564 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 565 566 // ...and a similar frame index for the caller-allocated save area 567 // that will be used to store the incoming registers. 568 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 569 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 570 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 571 572 // Store the FPR varargs in the reserved frame slots. (We store the 573 // GPRs as part of the prologue.) 574 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 575 SDValue MemOps[SystemZ::NumArgFPRs]; 576 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 577 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 578 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 579 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 580 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 581 &SystemZ::FP64BitRegClass); 582 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 583 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 584 MachinePointerInfo::getFixedStack(FI), 585 false, false, 0); 586 587 } 588 // Join the stores, which are independent of one another. 589 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 590 &MemOps[NumFixedFPRs], 591 SystemZ::NumArgFPRs - NumFixedFPRs); 592 } 593 } 594 595 return Chain; 596} 597 598SDValue 599SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 600 SmallVectorImpl<SDValue> &InVals) const { 601 SelectionDAG &DAG = CLI.DAG; 602 SDLoc &DL = CLI.DL; 603 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; 604 SmallVector<SDValue, 32> &OutVals = CLI.OutVals; 605 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; 606 SDValue Chain = CLI.Chain; 607 SDValue Callee = CLI.Callee; 608 bool &isTailCall = CLI.IsTailCall; 609 CallingConv::ID CallConv = CLI.CallConv; 610 bool IsVarArg = CLI.IsVarArg; 611 MachineFunction &MF = DAG.getMachineFunction(); 612 EVT PtrVT = getPointerTy(); 613 614 // SystemZ target does not yet support tail call optimization. 615 isTailCall = false; 616 617 // Analyze the operands of the call, assigning locations to each operand. 618 SmallVector<CCValAssign, 16> ArgLocs; 619 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 620 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 621 622 // Get a count of how many bytes are to be pushed on the stack. 623 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 624 625 // Mark the start of the call. 626 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true)); 627 628 // Copy argument values to their designated locations. 629 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 630 SmallVector<SDValue, 8> MemOpChains; 631 SDValue StackPtr; 632 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 633 CCValAssign &VA = ArgLocs[I]; 634 SDValue ArgValue = OutVals[I]; 635 636 if (VA.getLocInfo() == CCValAssign::Indirect) { 637 // Store the argument in a stack slot and pass its address. 638 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 639 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 640 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 641 MachinePointerInfo::getFixedStack(FI), 642 false, false, 0)); 643 ArgValue = SpillSlot; 644 } else 645 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 646 647 if (VA.isRegLoc()) 648 // Queue up the argument copies and emit them at the end. 649 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 650 else { 651 assert(VA.isMemLoc() && "Argument not register or memory"); 652 653 // Work out the address of the stack slot. Unpromoted ints and 654 // floats are passed as right-justified 8-byte values. 655 if (!StackPtr.getNode()) 656 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 657 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 658 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 659 Offset += 4; 660 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 661 DAG.getIntPtrConstant(Offset)); 662 663 // Emit the store. 664 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 665 MachinePointerInfo(), 666 false, false, 0)); 667 } 668 } 669 670 // Join the stores, which are independent of one another. 671 if (!MemOpChains.empty()) 672 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 673 &MemOpChains[0], MemOpChains.size()); 674 675 // Build a sequence of copy-to-reg nodes, chained and glued together. 676 SDValue Glue; 677 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 678 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 679 RegsToPass[I].second, Glue); 680 Glue = Chain.getValue(1); 681 } 682 683 // Accept direct calls by converting symbolic call addresses to the 684 // associated Target* opcodes. 685 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 686 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 687 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 688 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 689 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 690 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 691 } 692 693 // The first call operand is the chain and the second is the target address. 694 SmallVector<SDValue, 8> Ops; 695 Ops.push_back(Chain); 696 Ops.push_back(Callee); 697 698 // Add argument registers to the end of the list so that they are 699 // known live into the call. 700 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 701 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 702 RegsToPass[I].second.getValueType())); 703 704 // Glue the call to the argument copies, if any. 705 if (Glue.getNode()) 706 Ops.push_back(Glue); 707 708 // Emit the call. 709 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 710 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 711 Glue = Chain.getValue(1); 712 713 // Mark the end of the call, which is glued to the call itself. 714 Chain = DAG.getCALLSEQ_END(Chain, 715 DAG.getConstant(NumBytes, PtrVT, true), 716 DAG.getConstant(0, PtrVT, true), 717 Glue); 718 Glue = Chain.getValue(1); 719 720 // Assign locations to each value returned by this call. 721 SmallVector<CCValAssign, 16> RetLocs; 722 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 723 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 724 725 // Copy all of the result registers out of their specified physreg. 726 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 727 CCValAssign &VA = RetLocs[I]; 728 729 // Copy the value out, gluing the copy to the end of the call sequence. 730 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 731 VA.getLocVT(), Glue); 732 Chain = RetValue.getValue(1); 733 Glue = RetValue.getValue(2); 734 735 // Convert the value of the return register into the value that's 736 // being returned. 737 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 738 } 739 740 return Chain; 741} 742 743SDValue 744SystemZTargetLowering::LowerReturn(SDValue Chain, 745 CallingConv::ID CallConv, bool IsVarArg, 746 const SmallVectorImpl<ISD::OutputArg> &Outs, 747 const SmallVectorImpl<SDValue> &OutVals, 748 SDLoc DL, SelectionDAG &DAG) const { 749 MachineFunction &MF = DAG.getMachineFunction(); 750 751 // Assign locations to each returned value. 752 SmallVector<CCValAssign, 16> RetLocs; 753 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 754 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 755 756 // Quick exit for void returns 757 if (RetLocs.empty()) 758 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 759 760 // Copy the result values into the output registers. 761 SDValue Glue; 762 SmallVector<SDValue, 4> RetOps; 763 RetOps.push_back(Chain); 764 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 765 CCValAssign &VA = RetLocs[I]; 766 SDValue RetValue = OutVals[I]; 767 768 // Make the return register live on exit. 769 assert(VA.isRegLoc() && "Can only return in registers!"); 770 771 // Promote the value as required. 772 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 773 774 // Chain and glue the copies together. 775 unsigned Reg = VA.getLocReg(); 776 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 777 Glue = Chain.getValue(1); 778 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 779 } 780 781 // Update chain and glue. 782 RetOps[0] = Chain; 783 if (Glue.getNode()) 784 RetOps.push_back(Glue); 785 786 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 787 RetOps.data(), RetOps.size()); 788} 789 790// CC is a comparison that will be implemented using an integer or 791// floating-point comparison. Return the condition code mask for 792// a branch on true. In the integer case, CCMASK_CMP_UO is set for 793// unsigned comparisons and clear for signed ones. In the floating-point 794// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 795static unsigned CCMaskForCondCode(ISD::CondCode CC) { 796#define CONV(X) \ 797 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 798 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 799 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 800 801 switch (CC) { 802 default: 803 llvm_unreachable("Invalid integer condition!"); 804 805 CONV(EQ); 806 CONV(NE); 807 CONV(GT); 808 CONV(GE); 809 CONV(LT); 810 CONV(LE); 811 812 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 813 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 814 } 815#undef CONV 816} 817 818// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 819// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 820static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 821 SDValue &CmpOp0, SDValue &CmpOp1, 822 unsigned &CCMask) { 823 // For us to make any changes, it must a comparison between a single-use 824 // load and a constant. 825 if (!CmpOp0.hasOneUse() || 826 CmpOp0.getOpcode() != ISD::LOAD || 827 CmpOp1.getOpcode() != ISD::Constant) 828 return; 829 830 // We must have an 8- or 16-bit load. 831 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 832 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 833 if (NumBits != 8 && NumBits != 16) 834 return; 835 836 // The load must be an extending one and the constant must be within the 837 // range of the unextended value. 838 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 839 uint64_t Value = Constant->getZExtValue(); 840 uint64_t Mask = (1 << NumBits) - 1; 841 if (Load->getExtensionType() == ISD::SEXTLOAD) { 842 int64_t SignedValue = Constant->getSExtValue(); 843 if (uint64_t(SignedValue) + (1ULL << (NumBits - 1)) > Mask) 844 return; 845 // Unsigned comparison between two sign-extended values is equivalent 846 // to unsigned comparison between two zero-extended values. 847 if (IsUnsigned) 848 Value &= Mask; 849 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 850 CCMask == SystemZ::CCMASK_CMP_NE) 851 // Any choice of IsUnsigned is OK for equality comparisons. 852 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 853 // but since we use CLHHSI for zero extensions, it seems better 854 // to be consistent and do the same here. 855 Value &= Mask, IsUnsigned = true; 856 else if (NumBits == 8) { 857 // Try to treat the comparison as unsigned, so that we can use CLI. 858 // Adjust CCMask and Value as necessary. 859 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 860 // Test whether the high bit of the byte is set. 861 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 862 else if (SignedValue == -1 && CCMask == SystemZ::CCMASK_CMP_GT) 863 // Test whether the high bit of the byte is clear. 864 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 865 else 866 // No instruction exists for this combination. 867 return; 868 } 869 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 870 if (Value > Mask) 871 return; 872 // Signed comparison between two zero-extended values is equivalent 873 // to unsigned comparison. 874 IsUnsigned = true; 875 } else 876 return; 877 878 // Make sure that the first operand is an i32 of the right extension type. 879 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 880 if (CmpOp0.getValueType() != MVT::i32 || 881 Load->getExtensionType() != ExtType) 882 CmpOp0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 883 Load->getChain(), Load->getBasePtr(), 884 Load->getPointerInfo(), Load->getMemoryVT(), 885 Load->isVolatile(), Load->isNonTemporal(), 886 Load->getAlignment()); 887 888 // Make sure that the second operand is an i32 with the right value. 889 if (CmpOp1.getValueType() != MVT::i32 || 890 Value != Constant->getZExtValue()) 891 CmpOp1 = DAG.getConstant(Value, MVT::i32); 892} 893 894// Return true if a comparison described by CCMask, CmpOp0 and CmpOp1 895// is an equality comparison that is better implemented using unsigned 896// rather than signed comparison instructions. 897static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0, 898 SDValue CmpOp1, unsigned CCMask) { 899 // The test must be for equality or inequality. 900 if (CCMask != SystemZ::CCMASK_CMP_EQ && CCMask != SystemZ::CCMASK_CMP_NE) 901 return false; 902 903 if (CmpOp1.getOpcode() == ISD::Constant) { 904 uint64_t Value = cast<ConstantSDNode>(CmpOp1)->getSExtValue(); 905 906 // If we're comparing with memory, prefer unsigned comparisons for 907 // values that are in the unsigned 16-bit range but not the signed 908 // 16-bit range. We want to use CLFHSI and CLGHSI. 909 if (CmpOp0.hasOneUse() && 910 ISD::isNormalLoad(CmpOp0.getNode()) && 911 (Value >= 32768 && Value < 65536)) 912 return true; 913 914 // Use unsigned comparisons for values that are in the CLGFI range 915 // but not in the CGFI range. 916 if (CmpOp0.getValueType() == MVT::i64 && (Value >> 31) == 1) 917 return true; 918 919 return false; 920 } 921 922 // Prefer CL for zero-extended loads. 923 if (CmpOp1.getOpcode() == ISD::ZERO_EXTEND || 924 ISD::isZEXTLoad(CmpOp1.getNode())) 925 return true; 926 927 // ...and for "in-register" zero extensions. 928 if (CmpOp1.getOpcode() == ISD::AND && CmpOp1.getValueType() == MVT::i64) { 929 SDValue Mask = CmpOp1.getOperand(1); 930 if (Mask.getOpcode() == ISD::Constant && 931 cast<ConstantSDNode>(Mask)->getZExtValue() == 0xffffffff) 932 return true; 933 } 934 935 return false; 936} 937 938// Return a target node that compares CmpOp0 and CmpOp1. Set CCMask to the 939// 4-bit condition-code mask for CC. 940static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 941 ISD::CondCode CC, unsigned &CCMask) { 942 bool IsUnsigned = false; 943 CCMask = CCMaskForCondCode(CC); 944 if (!CmpOp0.getValueType().isFloatingPoint()) { 945 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 946 CCMask &= ~SystemZ::CCMASK_CMP_UO; 947 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 948 if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask)) 949 IsUnsigned = true; 950 } 951 952 SDLoc DL(CmpOp0); 953 return DAG.getNode((IsUnsigned ? SystemZISD::UCMP : SystemZISD::CMP), 954 DL, MVT::Glue, CmpOp0, CmpOp1); 955} 956 957// Lower a binary operation that produces two VT results, one in each 958// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 959// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 960// on the extended Op0 and (unextended) Op1. Store the even register result 961// in Even and the odd register result in Odd. 962static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 963 unsigned Extend, unsigned Opcode, 964 SDValue Op0, SDValue Op1, 965 SDValue &Even, SDValue &Odd) { 966 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 967 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 968 SDValue(In128, 0), Op1); 969 bool Is32Bit = is32Bit(VT); 970 SDValue SubReg0 = DAG.getTargetConstant(SystemZ::even128(Is32Bit), VT); 971 SDValue SubReg1 = DAG.getTargetConstant(SystemZ::odd128(Is32Bit), VT); 972 SDNode *Reg0 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 973 VT, Result, SubReg0); 974 SDNode *Reg1 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 975 VT, Result, SubReg1); 976 Even = SDValue(Reg0, 0); 977 Odd = SDValue(Reg1, 0); 978} 979 980SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 981 SDValue Chain = Op.getOperand(0); 982 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 983 SDValue CmpOp0 = Op.getOperand(2); 984 SDValue CmpOp1 = Op.getOperand(3); 985 SDValue Dest = Op.getOperand(4); 986 SDLoc DL(Op); 987 988 unsigned CCMask; 989 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); 990 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 991 Chain, DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 992} 993 994SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 995 SelectionDAG &DAG) const { 996 SDValue CmpOp0 = Op.getOperand(0); 997 SDValue CmpOp1 = Op.getOperand(1); 998 SDValue TrueOp = Op.getOperand(2); 999 SDValue FalseOp = Op.getOperand(3); 1000 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1001 SDLoc DL(Op); 1002 1003 unsigned CCMask; 1004 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); 1005 1006 SmallVector<SDValue, 4> Ops; 1007 Ops.push_back(TrueOp); 1008 Ops.push_back(FalseOp); 1009 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1010 Ops.push_back(Flags); 1011 1012 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1013 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1014} 1015 1016SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1017 SelectionDAG &DAG) const { 1018 SDLoc DL(Node); 1019 const GlobalValue *GV = Node->getGlobal(); 1020 int64_t Offset = Node->getOffset(); 1021 EVT PtrVT = getPointerTy(); 1022 Reloc::Model RM = TM.getRelocationModel(); 1023 CodeModel::Model CM = TM.getCodeModel(); 1024 1025 SDValue Result; 1026 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1027 // Make sure that the offset is aligned to a halfword. If it isn't, 1028 // create an "anchor" at the previous 12-bit boundary. 1029 // FIXME check whether there is a better way of handling this. 1030 if (Offset & 1) { 1031 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 1032 Offset & ~uint64_t(0xfff)); 1033 Offset &= 0xfff; 1034 } else { 1035 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Offset); 1036 Offset = 0; 1037 } 1038 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1039 } else { 1040 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1041 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1042 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1043 MachinePointerInfo::getGOT(), false, false, false, 0); 1044 } 1045 1046 // If there was a non-zero offset that we didn't fold, create an explicit 1047 // addition for it. 1048 if (Offset != 0) 1049 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1050 DAG.getConstant(Offset, PtrVT)); 1051 1052 return Result; 1053} 1054 1055SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1056 SelectionDAG &DAG) const { 1057 SDLoc DL(Node); 1058 const GlobalValue *GV = Node->getGlobal(); 1059 EVT PtrVT = getPointerTy(); 1060 TLSModel::Model model = TM.getTLSModel(GV); 1061 1062 if (model != TLSModel::LocalExec) 1063 llvm_unreachable("only local-exec TLS mode supported"); 1064 1065 // The high part of the thread pointer is in access register 0. 1066 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1067 DAG.getConstant(0, MVT::i32)); 1068 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1069 1070 // The low part of the thread pointer is in access register 1. 1071 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1072 DAG.getConstant(1, MVT::i32)); 1073 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1074 1075 // Merge them into a single 64-bit address. 1076 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1077 DAG.getConstant(32, PtrVT)); 1078 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1079 1080 // Get the offset of GA from the thread pointer. 1081 SystemZConstantPoolValue *CPV = 1082 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1083 1084 // Force the offset into the constant pool and load it from there. 1085 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1086 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1087 CPAddr, MachinePointerInfo::getConstantPool(), 1088 false, false, false, 0); 1089 1090 // Add the base and offset together. 1091 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1092} 1093 1094SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1095 SelectionDAG &DAG) const { 1096 SDLoc DL(Node); 1097 const BlockAddress *BA = Node->getBlockAddress(); 1098 int64_t Offset = Node->getOffset(); 1099 EVT PtrVT = getPointerTy(); 1100 1101 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1102 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1103 return Result; 1104} 1105 1106SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1107 SelectionDAG &DAG) const { 1108 SDLoc DL(JT); 1109 EVT PtrVT = getPointerTy(); 1110 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1111 1112 // Use LARL to load the address of the table. 1113 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1114} 1115 1116SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1117 SelectionDAG &DAG) const { 1118 SDLoc DL(CP); 1119 EVT PtrVT = getPointerTy(); 1120 1121 SDValue Result; 1122 if (CP->isMachineConstantPoolEntry()) 1123 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1124 CP->getAlignment()); 1125 else 1126 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1127 CP->getAlignment(), CP->getOffset()); 1128 1129 // Use LARL to load the address of the constant pool entry. 1130 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1131} 1132 1133SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1134 SelectionDAG &DAG) const { 1135 SDLoc DL(Op); 1136 SDValue In = Op.getOperand(0); 1137 EVT InVT = In.getValueType(); 1138 EVT ResVT = Op.getValueType(); 1139 1140 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1141 SDValue Shift32 = DAG.getConstant(32, MVT::i64); 1142 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1143 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1144 SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); 1145 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); 1146 SDNode *Out = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1147 MVT::f32, Out64, SubReg32); 1148 return SDValue(Out, 0); 1149 } 1150 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1151 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1152 SDNode *In64 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1153 MVT::f64, SDValue(U64, 0), In, SubReg32); 1154 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, SDValue(In64, 0)); 1155 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); 1156 SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1157 return Out; 1158 } 1159 llvm_unreachable("Unexpected bitcast combination"); 1160} 1161 1162SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1163 SelectionDAG &DAG) const { 1164 MachineFunction &MF = DAG.getMachineFunction(); 1165 SystemZMachineFunctionInfo *FuncInfo = 1166 MF.getInfo<SystemZMachineFunctionInfo>(); 1167 EVT PtrVT = getPointerTy(); 1168 1169 SDValue Chain = Op.getOperand(0); 1170 SDValue Addr = Op.getOperand(1); 1171 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1172 SDLoc DL(Op); 1173 1174 // The initial values of each field. 1175 const unsigned NumFields = 4; 1176 SDValue Fields[NumFields] = { 1177 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1178 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1179 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1180 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1181 }; 1182 1183 // Store each field into its respective slot. 1184 SDValue MemOps[NumFields]; 1185 unsigned Offset = 0; 1186 for (unsigned I = 0; I < NumFields; ++I) { 1187 SDValue FieldAddr = Addr; 1188 if (Offset != 0) 1189 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1190 DAG.getIntPtrConstant(Offset)); 1191 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1192 MachinePointerInfo(SV, Offset), 1193 false, false, 0); 1194 Offset += 8; 1195 } 1196 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1197} 1198 1199SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1200 SelectionDAG &DAG) const { 1201 SDValue Chain = Op.getOperand(0); 1202 SDValue DstPtr = Op.getOperand(1); 1203 SDValue SrcPtr = Op.getOperand(2); 1204 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1205 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1206 SDLoc DL(Op); 1207 1208 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1209 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1210 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1211} 1212 1213SDValue SystemZTargetLowering:: 1214lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1215 SDValue Chain = Op.getOperand(0); 1216 SDValue Size = Op.getOperand(1); 1217 SDLoc DL(Op); 1218 1219 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1220 1221 // Get a reference to the stack pointer. 1222 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1223 1224 // Get the new stack pointer value. 1225 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1226 1227 // Copy the new stack pointer back. 1228 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1229 1230 // The allocated data lives above the 160 bytes allocated for the standard 1231 // frame, plus any outgoing stack arguments. We don't know how much that 1232 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1233 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1234 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1235 1236 SDValue Ops[2] = { Result, Chain }; 1237 return DAG.getMergeValues(Ops, 2, DL); 1238} 1239 1240SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1241 SelectionDAG &DAG) const { 1242 EVT VT = Op.getValueType(); 1243 SDLoc DL(Op); 1244 assert(!is32Bit(VT) && "Only support 64-bit UMUL_LOHI"); 1245 1246 // UMUL_LOHI64 returns the low result in the odd register and the high 1247 // result in the even register. UMUL_LOHI is defined to return the 1248 // low half first, so the results are in reverse order. 1249 SDValue Ops[2]; 1250 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1251 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1252 return DAG.getMergeValues(Ops, 2, DL); 1253} 1254 1255SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1256 SelectionDAG &DAG) const { 1257 SDValue Op0 = Op.getOperand(0); 1258 SDValue Op1 = Op.getOperand(1); 1259 EVT VT = Op.getValueType(); 1260 SDLoc DL(Op); 1261 1262 // We use DSGF for 32-bit division. 1263 if (is32Bit(VT)) { 1264 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1265 Op1 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op1); 1266 } 1267 1268 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1269 // input is "don't care". The instruction returns the remainder in 1270 // the even register and the quotient in the odd register. 1271 SDValue Ops[2]; 1272 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::SDIVREM64, 1273 Op0, Op1, Ops[1], Ops[0]); 1274 return DAG.getMergeValues(Ops, 2, DL); 1275} 1276 1277SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1278 SelectionDAG &DAG) const { 1279 EVT VT = Op.getValueType(); 1280 SDLoc DL(Op); 1281 1282 // DL(G) uses a double-width dividend, so we need to clear the even 1283 // register in the GR128 input. The instruction returns the remainder 1284 // in the even register and the quotient in the odd register. 1285 SDValue Ops[2]; 1286 if (is32Bit(VT)) 1287 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1288 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1289 else 1290 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1291 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1292 return DAG.getMergeValues(Ops, 2, DL); 1293} 1294 1295SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1296 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1297 1298 // Get the known-zero masks for each operand. 1299 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1300 APInt KnownZero[2], KnownOne[2]; 1301 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1302 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1303 1304 // See if the upper 32 bits of one operand and the lower 32 bits of the 1305 // other are known zero. They are the low and high operands respectively. 1306 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1307 KnownZero[1].getZExtValue() }; 1308 unsigned High, Low; 1309 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1310 High = 1, Low = 0; 1311 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1312 High = 0, Low = 1; 1313 else 1314 return Op; 1315 1316 SDValue LowOp = Ops[Low]; 1317 SDValue HighOp = Ops[High]; 1318 1319 // If the high part is a constant, we're better off using IILH. 1320 if (HighOp.getOpcode() == ISD::Constant) 1321 return Op; 1322 1323 // If the low part is a constant that is outside the range of LHI, 1324 // then we're better off using IILF. 1325 if (LowOp.getOpcode() == ISD::Constant) { 1326 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1327 if (!isInt<16>(Value)) 1328 return Op; 1329 } 1330 1331 // Check whether the high part is an AND that doesn't change the 1332 // high 32 bits and just masks out low bits. We can skip it if so. 1333 if (HighOp.getOpcode() == ISD::AND && 1334 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1335 ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); 1336 uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; 1337 if ((Mask >> 32) == 0xffffffff) 1338 HighOp = HighOp.getOperand(0); 1339 } 1340 1341 // Take advantage of the fact that all GR32 operations only change the 1342 // low 32 bits by truncating Low to an i32 and inserting it directly 1343 // using a subreg. The interesting cases are those where the truncation 1344 // can be folded. 1345 SDLoc DL(Op); 1346 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1347 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1348 SDNode *Result = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1349 MVT::i64, HighOp, Low32, SubReg32); 1350 return SDValue(Result, 0); 1351} 1352 1353// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1354// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1355SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1356 SelectionDAG &DAG, 1357 unsigned Opcode) const { 1358 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1359 1360 // 32-bit operations need no code outside the main loop. 1361 EVT NarrowVT = Node->getMemoryVT(); 1362 EVT WideVT = MVT::i32; 1363 if (NarrowVT == WideVT) 1364 return Op; 1365 1366 int64_t BitSize = NarrowVT.getSizeInBits(); 1367 SDValue ChainIn = Node->getChain(); 1368 SDValue Addr = Node->getBasePtr(); 1369 SDValue Src2 = Node->getVal(); 1370 MachineMemOperand *MMO = Node->getMemOperand(); 1371 SDLoc DL(Node); 1372 EVT PtrVT = Addr.getValueType(); 1373 1374 // Convert atomic subtracts of constants into additions. 1375 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1376 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1377 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1378 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1379 } 1380 1381 // Get the address of the containing word. 1382 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1383 DAG.getConstant(-4, PtrVT)); 1384 1385 // Get the number of bits that the word must be rotated left in order 1386 // to bring the field to the top bits of a GR32. 1387 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1388 DAG.getConstant(3, PtrVT)); 1389 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1390 1391 // Get the complementing shift amount, for rotating a field in the top 1392 // bits back to its proper position. 1393 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1394 DAG.getConstant(0, WideVT), BitShift); 1395 1396 // Extend the source operand to 32 bits and prepare it for the inner loop. 1397 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1398 // operations require the source to be shifted in advance. (This shift 1399 // can be folded if the source is constant.) For AND and NAND, the lower 1400 // bits must be set, while for other opcodes they should be left clear. 1401 if (Opcode != SystemZISD::ATOMIC_SWAPW) 1402 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 1403 DAG.getConstant(32 - BitSize, WideVT)); 1404 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 1405 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 1406 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 1407 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 1408 1409 // Construct the ATOMIC_LOADW_* node. 1410 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1411 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 1412 DAG.getConstant(BitSize, WideVT) }; 1413 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 1414 array_lengthof(Ops), 1415 NarrowVT, MMO); 1416 1417 // Rotate the result of the final CS so that the field is in the lower 1418 // bits of a GR32, then truncate it. 1419 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 1420 DAG.getConstant(BitSize, WideVT)); 1421 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 1422 1423 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 1424 return DAG.getMergeValues(RetOps, 2, DL); 1425} 1426 1427// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 1428// into a fullword ATOMIC_CMP_SWAPW operation. 1429SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 1430 SelectionDAG &DAG) const { 1431 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1432 1433 // We have native support for 32-bit compare and swap. 1434 EVT NarrowVT = Node->getMemoryVT(); 1435 EVT WideVT = MVT::i32; 1436 if (NarrowVT == WideVT) 1437 return Op; 1438 1439 int64_t BitSize = NarrowVT.getSizeInBits(); 1440 SDValue ChainIn = Node->getOperand(0); 1441 SDValue Addr = Node->getOperand(1); 1442 SDValue CmpVal = Node->getOperand(2); 1443 SDValue SwapVal = Node->getOperand(3); 1444 MachineMemOperand *MMO = Node->getMemOperand(); 1445 SDLoc DL(Node); 1446 EVT PtrVT = Addr.getValueType(); 1447 1448 // Get the address of the containing word. 1449 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1450 DAG.getConstant(-4, PtrVT)); 1451 1452 // Get the number of bits that the word must be rotated left in order 1453 // to bring the field to the top bits of a GR32. 1454 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1455 DAG.getConstant(3, PtrVT)); 1456 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1457 1458 // Get the complementing shift amount, for rotating a field in the top 1459 // bits back to its proper position. 1460 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1461 DAG.getConstant(0, WideVT), BitShift); 1462 1463 // Construct the ATOMIC_CMP_SWAPW node. 1464 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1465 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 1466 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 1467 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 1468 VTList, Ops, array_lengthof(Ops), 1469 NarrowVT, MMO); 1470 return AtomicOp; 1471} 1472 1473SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 1474 SelectionDAG &DAG) const { 1475 MachineFunction &MF = DAG.getMachineFunction(); 1476 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1477 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 1478 SystemZ::R15D, Op.getValueType()); 1479} 1480 1481SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 1482 SelectionDAG &DAG) const { 1483 MachineFunction &MF = DAG.getMachineFunction(); 1484 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1485 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 1486 SystemZ::R15D, Op.getOperand(1)); 1487} 1488 1489SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 1490 SelectionDAG &DAG) const { 1491 switch (Op.getOpcode()) { 1492 case ISD::BR_CC: 1493 return lowerBR_CC(Op, DAG); 1494 case ISD::SELECT_CC: 1495 return lowerSELECT_CC(Op, DAG); 1496 case ISD::GlobalAddress: 1497 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 1498 case ISD::GlobalTLSAddress: 1499 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 1500 case ISD::BlockAddress: 1501 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 1502 case ISD::JumpTable: 1503 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 1504 case ISD::ConstantPool: 1505 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 1506 case ISD::BITCAST: 1507 return lowerBITCAST(Op, DAG); 1508 case ISD::VASTART: 1509 return lowerVASTART(Op, DAG); 1510 case ISD::VACOPY: 1511 return lowerVACOPY(Op, DAG); 1512 case ISD::DYNAMIC_STACKALLOC: 1513 return lowerDYNAMIC_STACKALLOC(Op, DAG); 1514 case ISD::UMUL_LOHI: 1515 return lowerUMUL_LOHI(Op, DAG); 1516 case ISD::SDIVREM: 1517 return lowerSDIVREM(Op, DAG); 1518 case ISD::UDIVREM: 1519 return lowerUDIVREM(Op, DAG); 1520 case ISD::OR: 1521 return lowerOR(Op, DAG); 1522 case ISD::ATOMIC_SWAP: 1523 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 1524 case ISD::ATOMIC_LOAD_ADD: 1525 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 1526 case ISD::ATOMIC_LOAD_SUB: 1527 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 1528 case ISD::ATOMIC_LOAD_AND: 1529 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 1530 case ISD::ATOMIC_LOAD_OR: 1531 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 1532 case ISD::ATOMIC_LOAD_XOR: 1533 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 1534 case ISD::ATOMIC_LOAD_NAND: 1535 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 1536 case ISD::ATOMIC_LOAD_MIN: 1537 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 1538 case ISD::ATOMIC_LOAD_MAX: 1539 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 1540 case ISD::ATOMIC_LOAD_UMIN: 1541 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 1542 case ISD::ATOMIC_LOAD_UMAX: 1543 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 1544 case ISD::ATOMIC_CMP_SWAP: 1545 return lowerATOMIC_CMP_SWAP(Op, DAG); 1546 case ISD::STACKSAVE: 1547 return lowerSTACKSAVE(Op, DAG); 1548 case ISD::STACKRESTORE: 1549 return lowerSTACKRESTORE(Op, DAG); 1550 default: 1551 llvm_unreachable("Unexpected node to lower"); 1552 } 1553} 1554 1555const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 1556#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 1557 switch (Opcode) { 1558 OPCODE(RET_FLAG); 1559 OPCODE(CALL); 1560 OPCODE(PCREL_WRAPPER); 1561 OPCODE(CMP); 1562 OPCODE(UCMP); 1563 OPCODE(BR_CCMASK); 1564 OPCODE(SELECT_CCMASK); 1565 OPCODE(ADJDYNALLOC); 1566 OPCODE(EXTRACT_ACCESS); 1567 OPCODE(UMUL_LOHI64); 1568 OPCODE(SDIVREM64); 1569 OPCODE(UDIVREM32); 1570 OPCODE(UDIVREM64); 1571 OPCODE(ATOMIC_SWAPW); 1572 OPCODE(ATOMIC_LOADW_ADD); 1573 OPCODE(ATOMIC_LOADW_SUB); 1574 OPCODE(ATOMIC_LOADW_AND); 1575 OPCODE(ATOMIC_LOADW_OR); 1576 OPCODE(ATOMIC_LOADW_XOR); 1577 OPCODE(ATOMIC_LOADW_NAND); 1578 OPCODE(ATOMIC_LOADW_MIN); 1579 OPCODE(ATOMIC_LOADW_MAX); 1580 OPCODE(ATOMIC_LOADW_UMIN); 1581 OPCODE(ATOMIC_LOADW_UMAX); 1582 OPCODE(ATOMIC_CMP_SWAPW); 1583 } 1584 return NULL; 1585#undef OPCODE 1586} 1587 1588//===----------------------------------------------------------------------===// 1589// Custom insertion 1590//===----------------------------------------------------------------------===// 1591 1592// Create a new basic block after MBB. 1593static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 1594 MachineFunction &MF = *MBB->getParent(); 1595 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1596 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 1597 return NewMBB; 1598} 1599 1600// Split MBB after MI and return the new block (the one that contains 1601// instructions after MI). 1602static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 1603 MachineBasicBlock *MBB) { 1604 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1605 NewMBB->splice(NewMBB->begin(), MBB, 1606 llvm::next(MachineBasicBlock::iterator(MI)), 1607 MBB->end()); 1608 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1609 return NewMBB; 1610} 1611 1612bool SystemZTargetLowering:: 1613convertPrevCompareToBranch(MachineBasicBlock *MBB, 1614 MachineBasicBlock::iterator MBBI, 1615 unsigned CCMask, MachineBasicBlock *Target) const { 1616 MachineBasicBlock::iterator Compare = MBBI; 1617 MachineBasicBlock::iterator Begin = MBB->begin(); 1618 do 1619 { 1620 if (Compare == Begin) 1621 return false; 1622 --Compare; 1623 } 1624 while (Compare->isDebugValue()); 1625 1626 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1627 unsigned FusedOpcode = TII->getCompareAndBranch(Compare->getOpcode()); 1628 if (!FusedOpcode) 1629 return false; 1630 1631 DebugLoc DL = Compare->getDebugLoc(); 1632 BuildMI(*MBB, MBBI, DL, TII->get(FusedOpcode)) 1633 .addOperand(Compare->getOperand(0)).addOperand(Compare->getOperand(1)) 1634 .addImm(CCMask).addMBB(Target); 1635 Compare->removeFromParent(); 1636 return true; 1637} 1638 1639// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 1640MachineBasicBlock * 1641SystemZTargetLowering::emitSelect(MachineInstr *MI, 1642 MachineBasicBlock *MBB) const { 1643 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1644 1645 unsigned DestReg = MI->getOperand(0).getReg(); 1646 unsigned TrueReg = MI->getOperand(1).getReg(); 1647 unsigned FalseReg = MI->getOperand(2).getReg(); 1648 unsigned CCMask = MI->getOperand(3).getImm(); 1649 DebugLoc DL = MI->getDebugLoc(); 1650 1651 MachineBasicBlock *StartMBB = MBB; 1652 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 1653 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 1654 1655 // StartMBB: 1656 // BRC CCMask, JoinMBB 1657 // # fallthrough to FalseMBB 1658 // 1659 // The original DAG glues comparisons to their uses, both to ensure 1660 // that no CC-clobbering instructions are inserted between them, and 1661 // to ensure that comparison results are not reused. This means that 1662 // this Select is the sole user of any preceding comparison instruction 1663 // and that we can try to use a fused compare and branch instead. 1664 MBB = StartMBB; 1665 if (!convertPrevCompareToBranch(MBB, MI, CCMask, JoinMBB)) 1666 BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(CCMask).addMBB(JoinMBB); 1667 MBB->addSuccessor(JoinMBB); 1668 MBB->addSuccessor(FalseMBB); 1669 1670 // FalseMBB: 1671 // # fallthrough to JoinMBB 1672 MBB = FalseMBB; 1673 MBB->addSuccessor(JoinMBB); 1674 1675 // JoinMBB: 1676 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 1677 // ... 1678 MBB = JoinMBB; 1679 BuildMI(*MBB, MBB->begin(), DL, TII->get(SystemZ::PHI), DestReg) 1680 .addReg(TrueReg).addMBB(StartMBB) 1681 .addReg(FalseReg).addMBB(FalseMBB); 1682 1683 MI->eraseFromParent(); 1684 return JoinMBB; 1685} 1686 1687// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 1688// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 1689// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 1690// BitSize is the width of the field in bits, or 0 if this is a partword 1691// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 1692// is one of the operands. Invert says whether the field should be 1693// inverted after performing BinOpcode (e.g. for NAND). 1694MachineBasicBlock * 1695SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 1696 MachineBasicBlock *MBB, 1697 unsigned BinOpcode, 1698 unsigned BitSize, 1699 bool Invert) const { 1700 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1701 MachineFunction &MF = *MBB->getParent(); 1702 MachineRegisterInfo &MRI = MF.getRegInfo(); 1703 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1704 bool IsSubWord = (BitSize < 32); 1705 1706 // Extract the operands. Base can be a register or a frame index. 1707 // Src2 can be a register or immediate. 1708 unsigned Dest = MI->getOperand(0).getReg(); 1709 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1710 int64_t Disp = MI->getOperand(2).getImm(); 1711 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 1712 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 1713 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 1714 DebugLoc DL = MI->getDebugLoc(); 1715 if (IsSubWord) 1716 BitSize = MI->getOperand(6).getImm(); 1717 1718 // Subword operations use 32-bit registers. 1719 const TargetRegisterClass *RC = (BitSize <= 32 ? 1720 &SystemZ::GR32BitRegClass : 1721 &SystemZ::GR64BitRegClass); 1722 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 1723 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 1724 1725 // Get the right opcodes for the displacement. 1726 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 1727 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 1728 assert(LOpcode && CSOpcode && "Displacement out of range"); 1729 1730 // Create virtual registers for temporary results. 1731 unsigned OrigVal = MRI.createVirtualRegister(RC); 1732 unsigned OldVal = MRI.createVirtualRegister(RC); 1733 unsigned NewVal = (BinOpcode || IsSubWord ? 1734 MRI.createVirtualRegister(RC) : Src2.getReg()); 1735 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 1736 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 1737 1738 // Insert a basic block for the main loop. 1739 MachineBasicBlock *StartMBB = MBB; 1740 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1741 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1742 1743 // StartMBB: 1744 // ... 1745 // %OrigVal = L Disp(%Base) 1746 // # fall through to LoopMMB 1747 MBB = StartMBB; 1748 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 1749 .addOperand(Base).addImm(Disp).addReg(0); 1750 MBB->addSuccessor(LoopMBB); 1751 1752 // LoopMBB: 1753 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 1754 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 1755 // %RotatedNewVal = OP %RotatedOldVal, %Src2 1756 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 1757 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 1758 // JNE LoopMBB 1759 // # fall through to DoneMMB 1760 MBB = LoopMBB; 1761 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 1762 .addReg(OrigVal).addMBB(StartMBB) 1763 .addReg(Dest).addMBB(LoopMBB); 1764 if (IsSubWord) 1765 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 1766 .addReg(OldVal).addReg(BitShift).addImm(0); 1767 if (Invert) { 1768 // Perform the operation normally and then invert every bit of the field. 1769 unsigned Tmp = MRI.createVirtualRegister(RC); 1770 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 1771 .addReg(RotatedOldVal).addOperand(Src2); 1772 if (BitSize < 32) 1773 // XILF with the upper BitSize bits set. 1774 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 1775 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 1776 else if (BitSize == 32) 1777 // XILF with every bit set. 1778 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 1779 .addReg(Tmp).addImm(~uint32_t(0)); 1780 else { 1781 // Use LCGR and add -1 to the result, which is more compact than 1782 // an XILF, XILH pair. 1783 unsigned Tmp2 = MRI.createVirtualRegister(RC); 1784 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 1785 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 1786 .addReg(Tmp2).addImm(-1); 1787 } 1788 } else if (BinOpcode) 1789 // A simply binary operation. 1790 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 1791 .addReg(RotatedOldVal).addOperand(Src2); 1792 else if (IsSubWord) 1793 // Use RISBG to rotate Src2 into position and use it to replace the 1794 // field in RotatedOldVal. 1795 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 1796 .addReg(RotatedOldVal).addReg(Src2.getReg()) 1797 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 1798 if (IsSubWord) 1799 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 1800 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 1801 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 1802 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 1803 BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB); 1804 MBB->addSuccessor(LoopMBB); 1805 MBB->addSuccessor(DoneMBB); 1806 1807 MI->eraseFromParent(); 1808 return DoneMBB; 1809} 1810 1811// Implement EmitInstrWithCustomInserter for pseudo 1812// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 1813// instruction that should be used to compare the current field with the 1814// minimum or maximum value. KeepOldMask is the BRC condition-code mask 1815// for when the current field should be kept. BitSize is the width of 1816// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 1817MachineBasicBlock * 1818SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 1819 MachineBasicBlock *MBB, 1820 unsigned CompareOpcode, 1821 unsigned KeepOldMask, 1822 unsigned BitSize) const { 1823 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1824 MachineFunction &MF = *MBB->getParent(); 1825 MachineRegisterInfo &MRI = MF.getRegInfo(); 1826 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1827 bool IsSubWord = (BitSize < 32); 1828 1829 // Extract the operands. Base can be a register or a frame index. 1830 unsigned Dest = MI->getOperand(0).getReg(); 1831 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1832 int64_t Disp = MI->getOperand(2).getImm(); 1833 unsigned Src2 = MI->getOperand(3).getReg(); 1834 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 1835 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 1836 DebugLoc DL = MI->getDebugLoc(); 1837 if (IsSubWord) 1838 BitSize = MI->getOperand(6).getImm(); 1839 1840 // Subword operations use 32-bit registers. 1841 const TargetRegisterClass *RC = (BitSize <= 32 ? 1842 &SystemZ::GR32BitRegClass : 1843 &SystemZ::GR64BitRegClass); 1844 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 1845 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 1846 1847 // Get the right opcodes for the displacement. 1848 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 1849 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 1850 assert(LOpcode && CSOpcode && "Displacement out of range"); 1851 1852 // Create virtual registers for temporary results. 1853 unsigned OrigVal = MRI.createVirtualRegister(RC); 1854 unsigned OldVal = MRI.createVirtualRegister(RC); 1855 unsigned NewVal = MRI.createVirtualRegister(RC); 1856 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 1857 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 1858 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 1859 1860 // Insert 3 basic blocks for the loop. 1861 MachineBasicBlock *StartMBB = MBB; 1862 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1863 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1864 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 1865 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 1866 1867 // StartMBB: 1868 // ... 1869 // %OrigVal = L Disp(%Base) 1870 // # fall through to LoopMMB 1871 MBB = StartMBB; 1872 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 1873 .addOperand(Base).addImm(Disp).addReg(0); 1874 MBB->addSuccessor(LoopMBB); 1875 1876 // LoopMBB: 1877 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 1878 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 1879 // CompareOpcode %RotatedOldVal, %Src2 1880 // BRC KeepOldMask, UpdateMBB 1881 MBB = LoopMBB; 1882 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 1883 .addReg(OrigVal).addMBB(StartMBB) 1884 .addReg(Dest).addMBB(UpdateMBB); 1885 if (IsSubWord) 1886 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 1887 .addReg(OldVal).addReg(BitShift).addImm(0); 1888 unsigned FusedOpcode = TII->getCompareAndBranch(CompareOpcode); 1889 if (FusedOpcode) 1890 BuildMI(MBB, DL, TII->get(FusedOpcode)) 1891 .addReg(RotatedOldVal).addReg(Src2) 1892 .addImm(KeepOldMask).addMBB(UpdateMBB); 1893 else { 1894 BuildMI(MBB, DL, TII->get(CompareOpcode)) 1895 .addReg(RotatedOldVal).addReg(Src2); 1896 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 1897 .addImm(KeepOldMask).addMBB(UpdateMBB); 1898 } 1899 MBB->addSuccessor(UpdateMBB); 1900 MBB->addSuccessor(UseAltMBB); 1901 1902 // UseAltMBB: 1903 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 1904 // # fall through to UpdateMMB 1905 MBB = UseAltMBB; 1906 if (IsSubWord) 1907 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 1908 .addReg(RotatedOldVal).addReg(Src2) 1909 .addImm(32).addImm(31 + BitSize).addImm(0); 1910 MBB->addSuccessor(UpdateMBB); 1911 1912 // UpdateMBB: 1913 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 1914 // [ %RotatedAltVal, UseAltMBB ] 1915 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 1916 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 1917 // JNE LoopMBB 1918 // # fall through to DoneMMB 1919 MBB = UpdateMBB; 1920 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 1921 .addReg(RotatedOldVal).addMBB(LoopMBB) 1922 .addReg(RotatedAltVal).addMBB(UseAltMBB); 1923 if (IsSubWord) 1924 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 1925 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 1926 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 1927 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 1928 BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB); 1929 MBB->addSuccessor(LoopMBB); 1930 MBB->addSuccessor(DoneMBB); 1931 1932 MI->eraseFromParent(); 1933 return DoneMBB; 1934} 1935 1936// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 1937// instruction MI. 1938MachineBasicBlock * 1939SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 1940 MachineBasicBlock *MBB) const { 1941 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1942 MachineFunction &MF = *MBB->getParent(); 1943 MachineRegisterInfo &MRI = MF.getRegInfo(); 1944 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1945 1946 // Extract the operands. Base can be a register or a frame index. 1947 unsigned Dest = MI->getOperand(0).getReg(); 1948 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1949 int64_t Disp = MI->getOperand(2).getImm(); 1950 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 1951 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 1952 unsigned BitShift = MI->getOperand(5).getReg(); 1953 unsigned NegBitShift = MI->getOperand(6).getReg(); 1954 int64_t BitSize = MI->getOperand(7).getImm(); 1955 DebugLoc DL = MI->getDebugLoc(); 1956 1957 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 1958 1959 // Get the right opcodes for the displacement. 1960 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 1961 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 1962 assert(LOpcode && CSOpcode && "Displacement out of range"); 1963 1964 // Create virtual registers for temporary results. 1965 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 1966 unsigned OldVal = MRI.createVirtualRegister(RC); 1967 unsigned CmpVal = MRI.createVirtualRegister(RC); 1968 unsigned SwapVal = MRI.createVirtualRegister(RC); 1969 unsigned StoreVal = MRI.createVirtualRegister(RC); 1970 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 1971 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 1972 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 1973 1974 // Insert 2 basic blocks for the loop. 1975 MachineBasicBlock *StartMBB = MBB; 1976 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1977 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1978 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 1979 1980 // StartMBB: 1981 // ... 1982 // %OrigOldVal = L Disp(%Base) 1983 // # fall through to LoopMMB 1984 MBB = StartMBB; 1985 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 1986 .addOperand(Base).addImm(Disp).addReg(0); 1987 MBB->addSuccessor(LoopMBB); 1988 1989 // LoopMBB: 1990 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 1991 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 1992 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 1993 // %Dest = RLL %OldVal, BitSize(%BitShift) 1994 // ^^ The low BitSize bits contain the field 1995 // of interest. 1996 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 1997 // ^^ Replace the upper 32-BitSize bits of the 1998 // comparison value with those that we loaded, 1999 // so that we can use a full word comparison. 2000 // CRJNE %Dest, %RetryCmpVal, DoneMBB 2001 // # Fall through to SetMBB 2002 MBB = LoopMBB; 2003 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2004 .addReg(OrigOldVal).addMBB(StartMBB) 2005 .addReg(RetryOldVal).addMBB(SetMBB); 2006 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 2007 .addReg(OrigCmpVal).addMBB(StartMBB) 2008 .addReg(RetryCmpVal).addMBB(SetMBB); 2009 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 2010 .addReg(OrigSwapVal).addMBB(StartMBB) 2011 .addReg(RetrySwapVal).addMBB(SetMBB); 2012 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 2013 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 2014 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 2015 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2016 BuildMI(MBB, DL, TII->get(SystemZ::CRJ)) 2017 .addReg(Dest).addReg(RetryCmpVal) 2018 .addImm(MaskNE).addMBB(DoneMBB); 2019 MBB->addSuccessor(DoneMBB); 2020 MBB->addSuccessor(SetMBB); 2021 2022 // SetMBB: 2023 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 2024 // ^^ Replace the upper 32-BitSize bits of the new 2025 // value with those that we loaded. 2026 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 2027 // ^^ Rotate the new field to its proper position. 2028 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 2029 // JNE LoopMBB 2030 // # fall through to ExitMMB 2031 MBB = SetMBB; 2032 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 2033 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2034 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 2035 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 2036 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2037 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2038 BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB); 2039 MBB->addSuccessor(LoopMBB); 2040 MBB->addSuccessor(DoneMBB); 2041 2042 MI->eraseFromParent(); 2043 return DoneMBB; 2044} 2045 2046// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2047// if the high register of the GR128 value must be cleared or false if 2048// it's "don't care". SubReg is subreg_odd32 when extending a GR32 2049// and subreg_odd when extending a GR64. 2050MachineBasicBlock * 2051SystemZTargetLowering::emitExt128(MachineInstr *MI, 2052 MachineBasicBlock *MBB, 2053 bool ClearEven, unsigned SubReg) const { 2054 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2055 MachineFunction &MF = *MBB->getParent(); 2056 MachineRegisterInfo &MRI = MF.getRegInfo(); 2057 DebugLoc DL = MI->getDebugLoc(); 2058 2059 unsigned Dest = MI->getOperand(0).getReg(); 2060 unsigned Src = MI->getOperand(1).getReg(); 2061 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2062 2063 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2064 if (ClearEven) { 2065 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2066 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2067 2068 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2069 .addImm(0); 2070 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2071 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_high); 2072 In128 = NewIn128; 2073 } 2074 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2075 .addReg(In128).addReg(Src).addImm(SubReg); 2076 2077 MI->eraseFromParent(); 2078 return MBB; 2079} 2080 2081MachineBasicBlock *SystemZTargetLowering:: 2082EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 2083 switch (MI->getOpcode()) { 2084 case SystemZ::Select32: 2085 case SystemZ::SelectF32: 2086 case SystemZ::Select64: 2087 case SystemZ::SelectF64: 2088 case SystemZ::SelectF128: 2089 return emitSelect(MI, MBB); 2090 2091 case SystemZ::AEXT128_64: 2092 return emitExt128(MI, MBB, false, SystemZ::subreg_low); 2093 case SystemZ::ZEXT128_32: 2094 return emitExt128(MI, MBB, true, SystemZ::subreg_low32); 2095 case SystemZ::ZEXT128_64: 2096 return emitExt128(MI, MBB, true, SystemZ::subreg_low); 2097 2098 case SystemZ::ATOMIC_SWAPW: 2099 return emitAtomicLoadBinary(MI, MBB, 0, 0); 2100 case SystemZ::ATOMIC_SWAP_32: 2101 return emitAtomicLoadBinary(MI, MBB, 0, 32); 2102 case SystemZ::ATOMIC_SWAP_64: 2103 return emitAtomicLoadBinary(MI, MBB, 0, 64); 2104 2105 case SystemZ::ATOMIC_LOADW_AR: 2106 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 2107 case SystemZ::ATOMIC_LOADW_AFI: 2108 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 2109 case SystemZ::ATOMIC_LOAD_AR: 2110 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 2111 case SystemZ::ATOMIC_LOAD_AHI: 2112 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 2113 case SystemZ::ATOMIC_LOAD_AFI: 2114 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 2115 case SystemZ::ATOMIC_LOAD_AGR: 2116 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 2117 case SystemZ::ATOMIC_LOAD_AGHI: 2118 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 2119 case SystemZ::ATOMIC_LOAD_AGFI: 2120 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 2121 2122 case SystemZ::ATOMIC_LOADW_SR: 2123 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 2124 case SystemZ::ATOMIC_LOAD_SR: 2125 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 2126 case SystemZ::ATOMIC_LOAD_SGR: 2127 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 2128 2129 case SystemZ::ATOMIC_LOADW_NR: 2130 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 2131 case SystemZ::ATOMIC_LOADW_NILH: 2132 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0); 2133 case SystemZ::ATOMIC_LOAD_NR: 2134 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 2135 case SystemZ::ATOMIC_LOAD_NILL32: 2136 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32); 2137 case SystemZ::ATOMIC_LOAD_NILH32: 2138 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32); 2139 case SystemZ::ATOMIC_LOAD_NILF32: 2140 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32); 2141 case SystemZ::ATOMIC_LOAD_NGR: 2142 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 2143 case SystemZ::ATOMIC_LOAD_NILL: 2144 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64); 2145 case SystemZ::ATOMIC_LOAD_NILH: 2146 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64); 2147 case SystemZ::ATOMIC_LOAD_NIHL: 2148 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); 2149 case SystemZ::ATOMIC_LOAD_NIHH: 2150 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); 2151 case SystemZ::ATOMIC_LOAD_NILF: 2152 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64); 2153 case SystemZ::ATOMIC_LOAD_NIHF: 2154 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); 2155 2156 case SystemZ::ATOMIC_LOADW_OR: 2157 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 2158 case SystemZ::ATOMIC_LOADW_OILH: 2159 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 0); 2160 case SystemZ::ATOMIC_LOAD_OR: 2161 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 2162 case SystemZ::ATOMIC_LOAD_OILL32: 2163 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL32, 32); 2164 case SystemZ::ATOMIC_LOAD_OILH32: 2165 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 32); 2166 case SystemZ::ATOMIC_LOAD_OILF32: 2167 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF32, 32); 2168 case SystemZ::ATOMIC_LOAD_OGR: 2169 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 2170 case SystemZ::ATOMIC_LOAD_OILL: 2171 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 64); 2172 case SystemZ::ATOMIC_LOAD_OILH: 2173 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 64); 2174 case SystemZ::ATOMIC_LOAD_OIHL: 2175 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL, 64); 2176 case SystemZ::ATOMIC_LOAD_OIHH: 2177 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH, 64); 2178 case SystemZ::ATOMIC_LOAD_OILF: 2179 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 64); 2180 case SystemZ::ATOMIC_LOAD_OIHF: 2181 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF, 64); 2182 2183 case SystemZ::ATOMIC_LOADW_XR: 2184 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 2185 case SystemZ::ATOMIC_LOADW_XILF: 2186 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 0); 2187 case SystemZ::ATOMIC_LOAD_XR: 2188 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 2189 case SystemZ::ATOMIC_LOAD_XILF32: 2190 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 32); 2191 case SystemZ::ATOMIC_LOAD_XGR: 2192 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 2193 case SystemZ::ATOMIC_LOAD_XILF: 2194 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 64); 2195 case SystemZ::ATOMIC_LOAD_XIHF: 2196 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); 2197 2198 case SystemZ::ATOMIC_LOADW_NRi: 2199 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 2200 case SystemZ::ATOMIC_LOADW_NILHi: 2201 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0, true); 2202 case SystemZ::ATOMIC_LOAD_NRi: 2203 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 2204 case SystemZ::ATOMIC_LOAD_NILL32i: 2205 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32, true); 2206 case SystemZ::ATOMIC_LOAD_NILH32i: 2207 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32, true); 2208 case SystemZ::ATOMIC_LOAD_NILF32i: 2209 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32, true); 2210 case SystemZ::ATOMIC_LOAD_NGRi: 2211 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 2212 case SystemZ::ATOMIC_LOAD_NILLi: 2213 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64, true); 2214 case SystemZ::ATOMIC_LOAD_NILHi: 2215 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64, true); 2216 case SystemZ::ATOMIC_LOAD_NIHLi: 2217 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); 2218 case SystemZ::ATOMIC_LOAD_NIHHi: 2219 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); 2220 case SystemZ::ATOMIC_LOAD_NILFi: 2221 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64, true); 2222 case SystemZ::ATOMIC_LOAD_NIHFi: 2223 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); 2224 2225 case SystemZ::ATOMIC_LOADW_MIN: 2226 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2227 SystemZ::CCMASK_CMP_LE, 0); 2228 case SystemZ::ATOMIC_LOAD_MIN_32: 2229 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2230 SystemZ::CCMASK_CMP_LE, 32); 2231 case SystemZ::ATOMIC_LOAD_MIN_64: 2232 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2233 SystemZ::CCMASK_CMP_LE, 64); 2234 2235 case SystemZ::ATOMIC_LOADW_MAX: 2236 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2237 SystemZ::CCMASK_CMP_GE, 0); 2238 case SystemZ::ATOMIC_LOAD_MAX_32: 2239 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2240 SystemZ::CCMASK_CMP_GE, 32); 2241 case SystemZ::ATOMIC_LOAD_MAX_64: 2242 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2243 SystemZ::CCMASK_CMP_GE, 64); 2244 2245 case SystemZ::ATOMIC_LOADW_UMIN: 2246 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2247 SystemZ::CCMASK_CMP_LE, 0); 2248 case SystemZ::ATOMIC_LOAD_UMIN_32: 2249 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2250 SystemZ::CCMASK_CMP_LE, 32); 2251 case SystemZ::ATOMIC_LOAD_UMIN_64: 2252 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2253 SystemZ::CCMASK_CMP_LE, 64); 2254 2255 case SystemZ::ATOMIC_LOADW_UMAX: 2256 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2257 SystemZ::CCMASK_CMP_GE, 0); 2258 case SystemZ::ATOMIC_LOAD_UMAX_32: 2259 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2260 SystemZ::CCMASK_CMP_GE, 32); 2261 case SystemZ::ATOMIC_LOAD_UMAX_64: 2262 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2263 SystemZ::CCMASK_CMP_GE, 64); 2264 2265 case SystemZ::ATOMIC_CMP_SWAPW: 2266 return emitAtomicCmpSwapW(MI, MBB); 2267 case SystemZ::BRC: 2268 // The original DAG glues comparisons to their uses, both to ensure 2269 // that no CC-clobbering instructions are inserted between them, and 2270 // to ensure that comparison results are not reused. This means that 2271 // a BRC is the sole user of a preceding comparison and that we can 2272 // try to use a fused compare and branch instead. 2273 if (convertPrevCompareToBranch(MBB, MI, MI->getOperand(0).getImm(), 2274 MI->getOperand(1).getMBB())) 2275 MI->eraseFromParent(); 2276 return MBB; 2277 default: 2278 llvm_unreachable("Unexpected instr type to insert"); 2279 } 2280} 2281