SystemZISelLowering.cpp revision 2a2a323488a31fbdb3524f7f288b8e5c3fc3b7c3
15d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 25d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// 35d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// The LLVM Compiler Infrastructure 45d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// 55d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// This file is distributed under the University of Illinois Open Source 65d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// License. See LICENSE.TXT for details. 75c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu// 85c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu//===----------------------------------------------------------------------===// 95d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// 105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// This file implements the SystemZTargetLowering class. 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)//===----------------------------------------------------------------------===// 135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#define DEBUG_TYPE "systemz-lower" 155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "SystemZISelLowering.h" 175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "SystemZCallingConv.h" 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "SystemZConstantPoolValue.h" 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "SystemZMachineFunctionInfo.h" 205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "SystemZTargetMachine.h" 215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "llvm/CodeGen/CallingConvLower.h" 225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "llvm/CodeGen/MachineInstrBuilder.h" 235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "llvm/CodeGen/MachineRegisterInfo.h" 245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) 265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include <cctype> 275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 285c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuusing namespace llvm; 295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)namespace { 315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// Represents a sequence for extracting a 0/1 value from an IPM result: 325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// (((X ^ XORValue) + AddValue) >> Bit) 335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)struct IPMConversion { 345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) int64_t XORValue; 385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) int64_t AddValue; 395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) unsigned Bit; 405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)}; 415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// Classify VT as either 32 or 64 bit. 445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)static bool is32Bit(EVT VT) { 455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) switch (VT.getSimpleVT().SimpleTy) { 465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) case MVT::i32: 475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return true; 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) case MVT::i64: 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return false; 505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) default: 515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) llvm_unreachable("Unsupported type"); 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 535c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu} 545c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 555c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu// Return a version of MachineOperand that can be safely used before the 565c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu// final use. 575c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liustatic MachineOperand earlyUseOperand(MachineOperand Op) { 585c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if (Op.isReg()) 595c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu Op.setIsKill(false); 605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return Op; 615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) : TargetLowering(tm, new TargetLoweringObjectFileELF()), 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Subtarget(*tm.getSubtargetImpl()), TM(tm) { 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MVT PtrVT = getPointerTy(); 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Set up the register classes. 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (Subtarget.hasHighWord()) 705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else 725c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 735c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 745c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 755c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 765c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 775c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 785c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // Compute derived properties from the register classes 795c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu computeRegisterProperties(); 805c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 815c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // Set up special registers. 825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setExceptionPointerRegister(SystemZ::R6D); 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setExceptionSelectorRegister(SystemZ::R7D); 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setStackPointerRegisterToSaveRestore(SystemZ::R15D); 855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // TODO: It may be better to default to latency-oriented scheduling, however 875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // LLVM's current latency-oriented scheduler can't handle physreg definitions 885c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // such as SystemZ has with CC, so set this to the register-pressure 895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // scheduler, because it can. 905c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setSchedulingPreference(Sched::RegPressure); 915c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setBooleanContents(ZeroOrOneBooleanContent); 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Instructions are strings of 2-byte aligned 2-byte values. 965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setMinFunctionAlignment(2); 975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Handle operations that are handled in a similar way for all types. 99010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 100010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) I <= MVT::LAST_FP_VALUETYPE; 101010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) ++I) { 102010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) MVT VT = MVT::SimpleValueType(I); 1035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (isTypeLegal(VT)) { 1045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Lower SET_CC into an IPM-based sequence. 1055c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::SETCC, VT, Custom); 1065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 1085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SELECT, VT, Expand); 1095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Lower SELECT_CC and BR_CC into separate comparisons and branches. 1115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SELECT_CC, VT, Custom); 1125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::BR_CC, VT, Custom); 1135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 1145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 1155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Expand jump table branches as address arithmetic followed by an 1175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // indirect jump. 1185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::BR_JT, MVT::Other, Expand); 1195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Expand BRCOND into a BR_CC (see above). 1215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::BRCOND, MVT::Other, Expand); 1225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 123010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) // Handle integer types. 124010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 1255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) I <= MVT::LAST_INTEGER_VALUETYPE; 1265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ++I) { 1275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MVT VT = MVT::SimpleValueType(I); 1285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (isTypeLegal(VT)) { 1295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Expand individual DIV and REMs into DIVREMs. 1305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SDIV, VT, Expand); 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UDIV, VT, Expand); 1325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SREM, VT, Expand); 1335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UREM, VT, Expand); 1345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SDIVREM, VT, Custom); 1355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UDIVREM, VT, Custom); 1365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 1385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // FIXME: probably much too conservative. 1395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 1405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 1415c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 1425c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // No special instructions for these. 1435c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::CTPOP, VT, Expand); 1445c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::CTTZ, VT, Expand); 1455c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 1465c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 1475c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::ROTR, VT, Expand); 1485c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 1495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Use *MUL_LOHI where possible instead of MULH*. 1505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::MULHS, VT, Expand); 1515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::MULHU, VT, Expand); 1525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SMUL_LOHI, VT, Custom); 1535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UMUL_LOHI, VT, Custom); 1545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We have instructions for signed but not unsigned FP conversion. 1565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FP_TO_UINT, VT, Expand); 1575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 1585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 1595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Type legalization will convert 8- and 16-bit atomic operations into 1615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // forms that operate on i32s (but still keeping the original memory VT). 1625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Lower them into full i32 operations. 1635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 164a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 165a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 166a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 167a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 168a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 169a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 170a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 171a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 172a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 173a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 174a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 175a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 1765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We have instructions for signed but not unsigned FP conversion. 1775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Handle unsigned 32-bit types as signed 64-bit types. 1785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 1795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 1805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We have native support for a 64-bit CTLZ, via FLOGR. 1825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::CTLZ, MVT::i32, Promote); 1835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::CTLZ, MVT::i64, Legal); 1845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Give LowerOperation the chance to replace 64-bit ORs with subregs. 1865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::OR, MVT::i64, Custom); 1875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // FIXME: Can we support these natively? 1895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 1905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 1915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 1925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We have native instructions for i8, i16 and i32 extensions, but not i1. 1945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 1955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 1965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 1975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 1985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Handle the various types of symbolic address. 2005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::ConstantPool, PtrVT, Custom); 2015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 2025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 2035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::BlockAddress, PtrVT, Custom); 2045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::JumpTable, PtrVT, Custom); 2055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We need to handle dynamic allocations specially because of the 2075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // 160-byte area at the bottom of the stack. 2085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 2095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Use custom expanders so that we can force the function to use 2115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // a frame pointer. 2125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 2135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 2145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Handle prefetches with PFD or PFDRL. 2165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 2175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Handle floating-point types. 2195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for (unsigned I = MVT::FIRST_FP_VALUETYPE; 2205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) I <= MVT::LAST_FP_VALUETYPE; 2215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ++I) { 2225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MVT VT = MVT::SimpleValueType(I); 2235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (isTypeLegal(VT)) { 2245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We can use FI for FRINT. 2255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FRINT, VT, Legal); 2265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We can use the extended form of FI for other rounding operations. 2285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (Subtarget.hasFPExtension()) { 2295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FNEARBYINT, VT, Legal); 2305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FFLOOR, VT, Legal); 2315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FCEIL, VT, Legal); 2325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FTRUNC, VT, Legal); 2335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FROUND, VT, Legal); 2345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 2355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // No special instructions for these. 2375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FSIN, VT, Expand); 2385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FCOS, VT, Expand); 2395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FREM, VT, Expand); 2405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 2415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 2425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We have fused multiply-addition for f32 and f64 but not f128. 2445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FMA, MVT::f32, Legal); 2455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FMA, MVT::f64, Legal); 2465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::FMA, MVT::f128, Expand); 2475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Needed so that we don't try to implement f128 constant loads using 2495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // a load-and-extend of a f80 constant (in cases where the constant 2505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // would fit in an f80). 2515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 2525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Floating-point truncation and stores need to be done separately. 2545c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setTruncStoreAction(MVT::f64, MVT::f32, Expand); 2555c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setTruncStoreAction(MVT::f128, MVT::f32, Expand); 2565c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setTruncStoreAction(MVT::f128, MVT::f64, Expand); 2575c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 2585c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // We have 64-bit FPR<->GPR moves, but need special handling for 2595c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu // 32-bit forms. 2605c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu setOperationAction(ISD::BITCAST, MVT::i32, Custom); 2615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::BITCAST, MVT::f32, Custom); 2625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // VASTART and VACOPY need to deal with the SystemZ-specific varargs 2645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // structure, but VAEND is a no-op. 2655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::VASTART, MVT::Other, Custom); 2665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::VACOPY, MVT::Other, Custom); 2675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) setOperationAction(ISD::VAEND, MVT::Other, Expand); 2685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // We want to use MVC in preference to even a single load/store pair. 2705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MaxStoresPerMemcpy = 0; 2715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MaxStoresPerMemcpyOptSize = 0; 2725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // The main memset sequence is a byte store followed by an MVC. 2745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Two STC or MV..I stores win over that, but the kind of fused stores 2755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // generated by target-independent code don't when the byte value is 2765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 2775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // than "STC;MVC". Handle the choice in target-specific code instead. 2785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MaxStoresPerMemset = 0; 2795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) MaxStoresPerMemsetOptSize = 0; 2805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 2815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)EVT SystemZTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { 2835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (!VT.isVector()) 2845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return MVT::i32; 2855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return VT.changeVectorElementTypeToInteger(); 2865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 2875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 289 VT = VT.getScalarType(); 290 291 if (!VT.isSimple()) 292 return false; 293 294 switch (VT.getSimpleVT().SimpleTy) { 295 case MVT::f32: 296 case MVT::f64: 297 return true; 298 case MVT::f128: 299 return false; 300 default: 301 break; 302 } 303 304 return false; 305} 306 307bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 308 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 309 return Imm.isZero() || Imm.isNegZero(); 310} 311 312bool SystemZTargetLowering::allowsUnalignedMemoryAccesses(EVT VT, 313 bool *Fast) const { 314 // Unaligned accesses should never be slower than the expanded version. 315 // We check specifically for aligned accesses in the few cases where 316 // they are required. 317 if (Fast) 318 *Fast = true; 319 return true; 320} 321 322bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 323 Type *Ty) const { 324 // Punt on globals for now, although they can be used in limited 325 // RELATIVE LONG cases. 326 if (AM.BaseGV) 327 return false; 328 329 // Require a 20-bit signed offset. 330 if (!isInt<20>(AM.BaseOffs)) 331 return false; 332 333 // Indexing is OK but no scale factor can be applied. 334 return AM.Scale == 0 || AM.Scale == 1; 335} 336 337bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 338 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 339 return false; 340 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 341 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 342 return FromBits > ToBits; 343} 344 345bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 346 if (!FromVT.isInteger() || !ToVT.isInteger()) 347 return false; 348 unsigned FromBits = FromVT.getSizeInBits(); 349 unsigned ToBits = ToVT.getSizeInBits(); 350 return FromBits > ToBits; 351} 352 353//===----------------------------------------------------------------------===// 354// Inline asm support 355//===----------------------------------------------------------------------===// 356 357TargetLowering::ConstraintType 358SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 359 if (Constraint.size() == 1) { 360 switch (Constraint[0]) { 361 case 'a': // Address register 362 case 'd': // Data register (equivalent to 'r') 363 case 'f': // Floating-point register 364 case 'h': // High-part register 365 case 'r': // General-purpose register 366 return C_RegisterClass; 367 368 case 'Q': // Memory with base and unsigned 12-bit displacement 369 case 'R': // Likewise, plus an index 370 case 'S': // Memory with base and signed 20-bit displacement 371 case 'T': // Likewise, plus an index 372 case 'm': // Equivalent to 'T'. 373 return C_Memory; 374 375 case 'I': // Unsigned 8-bit constant 376 case 'J': // Unsigned 12-bit constant 377 case 'K': // Signed 16-bit constant 378 case 'L': // Signed 20-bit displacement (on all targets we support) 379 case 'M': // 0x7fffffff 380 return C_Other; 381 382 default: 383 break; 384 } 385 } 386 return TargetLowering::getConstraintType(Constraint); 387} 388 389TargetLowering::ConstraintWeight SystemZTargetLowering:: 390getSingleConstraintMatchWeight(AsmOperandInfo &info, 391 const char *constraint) const { 392 ConstraintWeight weight = CW_Invalid; 393 Value *CallOperandVal = info.CallOperandVal; 394 // If we don't have a value, we can't do a match, 395 // but allow it at the lowest weight. 396 if (CallOperandVal == NULL) 397 return CW_Default; 398 Type *type = CallOperandVal->getType(); 399 // Look at the constraint type. 400 switch (*constraint) { 401 default: 402 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 403 break; 404 405 case 'a': // Address register 406 case 'd': // Data register (equivalent to 'r') 407 case 'h': // High-part register 408 case 'r': // General-purpose register 409 if (CallOperandVal->getType()->isIntegerTy()) 410 weight = CW_Register; 411 break; 412 413 case 'f': // Floating-point register 414 if (type->isFloatingPointTy()) 415 weight = CW_Register; 416 break; 417 418 case 'I': // Unsigned 8-bit constant 419 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 420 if (isUInt<8>(C->getZExtValue())) 421 weight = CW_Constant; 422 break; 423 424 case 'J': // Unsigned 12-bit constant 425 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 426 if (isUInt<12>(C->getZExtValue())) 427 weight = CW_Constant; 428 break; 429 430 case 'K': // Signed 16-bit constant 431 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 432 if (isInt<16>(C->getSExtValue())) 433 weight = CW_Constant; 434 break; 435 436 case 'L': // Signed 20-bit displacement (on all targets we support) 437 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 438 if (isInt<20>(C->getSExtValue())) 439 weight = CW_Constant; 440 break; 441 442 case 'M': // 0x7fffffff 443 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 444 if (C->getZExtValue() == 0x7fffffff) 445 weight = CW_Constant; 446 break; 447 } 448 return weight; 449} 450 451// Parse a "{tNNN}" register constraint for which the register type "t" 452// has already been verified. MC is the class associated with "t" and 453// Map maps 0-based register numbers to LLVM register numbers. 454static std::pair<unsigned, const TargetRegisterClass *> 455parseRegisterNumber(const std::string &Constraint, 456 const TargetRegisterClass *RC, const unsigned *Map) { 457 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 458 if (isdigit(Constraint[2])) { 459 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 460 unsigned Index = atoi(Suffix.c_str()); 461 if (Index < 16 && Map[Index]) 462 return std::make_pair(Map[Index], RC); 463 } 464 return std::make_pair(0u, static_cast<TargetRegisterClass*>(0)); 465} 466 467std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 468getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 469 if (Constraint.size() == 1) { 470 // GCC Constraint Letters 471 switch (Constraint[0]) { 472 default: break; 473 case 'd': // Data register (equivalent to 'r') 474 case 'r': // General-purpose register 475 if (VT == MVT::i64) 476 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 477 else if (VT == MVT::i128) 478 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 479 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 480 481 case 'a': // Address register 482 if (VT == MVT::i64) 483 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 484 else if (VT == MVT::i128) 485 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 486 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 487 488 case 'h': // High-part register (an LLVM extension) 489 return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 490 491 case 'f': // Floating-point register 492 if (VT == MVT::f64) 493 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 494 else if (VT == MVT::f128) 495 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 496 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 497 } 498 } 499 if (Constraint[0] == '{') { 500 // We need to override the default register parsing for GPRs and FPRs 501 // because the interpretation depends on VT. The internal names of 502 // the registers are also different from the external names 503 // (F0D and F0S instead of F0, etc.). 504 if (Constraint[1] == 'r') { 505 if (VT == MVT::i32) 506 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 507 SystemZMC::GR32Regs); 508 if (VT == MVT::i128) 509 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 510 SystemZMC::GR128Regs); 511 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 512 SystemZMC::GR64Regs); 513 } 514 if (Constraint[1] == 'f') { 515 if (VT == MVT::f32) 516 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 517 SystemZMC::FP32Regs); 518 if (VT == MVT::f128) 519 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 520 SystemZMC::FP128Regs); 521 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 522 SystemZMC::FP64Regs); 523 } 524 } 525 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 526} 527 528void SystemZTargetLowering:: 529LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 530 std::vector<SDValue> &Ops, 531 SelectionDAG &DAG) const { 532 // Only support length 1 constraints for now. 533 if (Constraint.length() == 1) { 534 switch (Constraint[0]) { 535 case 'I': // Unsigned 8-bit constant 536 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 537 if (isUInt<8>(C->getZExtValue())) 538 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 539 Op.getValueType())); 540 return; 541 542 case 'J': // Unsigned 12-bit constant 543 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 544 if (isUInt<12>(C->getZExtValue())) 545 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 546 Op.getValueType())); 547 return; 548 549 case 'K': // Signed 16-bit constant 550 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 551 if (isInt<16>(C->getSExtValue())) 552 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 553 Op.getValueType())); 554 return; 555 556 case 'L': // Signed 20-bit displacement (on all targets we support) 557 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 558 if (isInt<20>(C->getSExtValue())) 559 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 560 Op.getValueType())); 561 return; 562 563 case 'M': // 0x7fffffff 564 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 565 if (C->getZExtValue() == 0x7fffffff) 566 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 567 Op.getValueType())); 568 return; 569 } 570 } 571 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 572} 573 574//===----------------------------------------------------------------------===// 575// Calling conventions 576//===----------------------------------------------------------------------===// 577 578#include "SystemZGenCallingConv.inc" 579 580bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 581 Type *ToType) const { 582 return isTruncateFree(FromType, ToType); 583} 584 585bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 586 if (!CI->isTailCall()) 587 return false; 588 return true; 589} 590 591// Value is a value that has been passed to us in the location described by VA 592// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 593// any loads onto Chain. 594static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 595 CCValAssign &VA, SDValue Chain, 596 SDValue Value) { 597 // If the argument has been promoted from a smaller type, insert an 598 // assertion to capture this. 599 if (VA.getLocInfo() == CCValAssign::SExt) 600 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 601 DAG.getValueType(VA.getValVT())); 602 else if (VA.getLocInfo() == CCValAssign::ZExt) 603 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 604 DAG.getValueType(VA.getValVT())); 605 606 if (VA.isExtInLoc()) 607 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 608 else if (VA.getLocInfo() == CCValAssign::Indirect) 609 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 610 MachinePointerInfo(), false, false, false, 0); 611 else 612 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 613 return Value; 614} 615 616// Value is a value of type VA.getValVT() that we need to copy into 617// the location described by VA. Return a copy of Value converted to 618// VA.getValVT(). The caller is responsible for handling indirect values. 619static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 620 CCValAssign &VA, SDValue Value) { 621 switch (VA.getLocInfo()) { 622 case CCValAssign::SExt: 623 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 624 case CCValAssign::ZExt: 625 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 626 case CCValAssign::AExt: 627 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 628 case CCValAssign::Full: 629 return Value; 630 default: 631 llvm_unreachable("Unhandled getLocInfo()"); 632 } 633} 634 635SDValue SystemZTargetLowering:: 636LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 637 const SmallVectorImpl<ISD::InputArg> &Ins, 638 SDLoc DL, SelectionDAG &DAG, 639 SmallVectorImpl<SDValue> &InVals) const { 640 MachineFunction &MF = DAG.getMachineFunction(); 641 MachineFrameInfo *MFI = MF.getFrameInfo(); 642 MachineRegisterInfo &MRI = MF.getRegInfo(); 643 SystemZMachineFunctionInfo *FuncInfo = 644 MF.getInfo<SystemZMachineFunctionInfo>(); 645 const SystemZFrameLowering *TFL = 646 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 647 648 // Assign locations to all of the incoming arguments. 649 SmallVector<CCValAssign, 16> ArgLocs; 650 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 651 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 652 653 unsigned NumFixedGPRs = 0; 654 unsigned NumFixedFPRs = 0; 655 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 656 SDValue ArgValue; 657 CCValAssign &VA = ArgLocs[I]; 658 EVT LocVT = VA.getLocVT(); 659 if (VA.isRegLoc()) { 660 // Arguments passed in registers 661 const TargetRegisterClass *RC; 662 switch (LocVT.getSimpleVT().SimpleTy) { 663 default: 664 // Integers smaller than i64 should be promoted to i64. 665 llvm_unreachable("Unexpected argument type"); 666 case MVT::i32: 667 NumFixedGPRs += 1; 668 RC = &SystemZ::GR32BitRegClass; 669 break; 670 case MVT::i64: 671 NumFixedGPRs += 1; 672 RC = &SystemZ::GR64BitRegClass; 673 break; 674 case MVT::f32: 675 NumFixedFPRs += 1; 676 RC = &SystemZ::FP32BitRegClass; 677 break; 678 case MVT::f64: 679 NumFixedFPRs += 1; 680 RC = &SystemZ::FP64BitRegClass; 681 break; 682 } 683 684 unsigned VReg = MRI.createVirtualRegister(RC); 685 MRI.addLiveIn(VA.getLocReg(), VReg); 686 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 687 } else { 688 assert(VA.isMemLoc() && "Argument not register or memory"); 689 690 // Create the frame index object for this incoming parameter. 691 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 692 VA.getLocMemOffset(), true); 693 694 // Create the SelectionDAG nodes corresponding to a load 695 // from this parameter. Unpromoted ints and floats are 696 // passed as right-justified 8-byte values. 697 EVT PtrVT = getPointerTy(); 698 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 699 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 700 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 701 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 702 MachinePointerInfo::getFixedStack(FI), 703 false, false, false, 0); 704 } 705 706 // Convert the value of the argument register into the value that's 707 // being passed. 708 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 709 } 710 711 if (IsVarArg) { 712 // Save the number of non-varargs registers for later use by va_start, etc. 713 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 714 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 715 716 // Likewise the address (in the form of a frame index) of where the 717 // first stack vararg would be. The 1-byte size here is arbitrary. 718 int64_t StackSize = CCInfo.getNextStackOffset(); 719 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 720 721 // ...and a similar frame index for the caller-allocated save area 722 // that will be used to store the incoming registers. 723 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 724 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 725 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 726 727 // Store the FPR varargs in the reserved frame slots. (We store the 728 // GPRs as part of the prologue.) 729 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 730 SDValue MemOps[SystemZ::NumArgFPRs]; 731 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 732 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 733 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 734 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 735 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 736 &SystemZ::FP64BitRegClass); 737 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 738 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 739 MachinePointerInfo::getFixedStack(FI), 740 false, false, 0); 741 742 } 743 // Join the stores, which are independent of one another. 744 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 745 &MemOps[NumFixedFPRs], 746 SystemZ::NumArgFPRs - NumFixedFPRs); 747 } 748 } 749 750 return Chain; 751} 752 753static bool canUseSiblingCall(CCState ArgCCInfo, 754 SmallVectorImpl<CCValAssign> &ArgLocs) { 755 // Punt if there are any indirect or stack arguments, or if the call 756 // needs the call-saved argument register R6. 757 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 758 CCValAssign &VA = ArgLocs[I]; 759 if (VA.getLocInfo() == CCValAssign::Indirect) 760 return false; 761 if (!VA.isRegLoc()) 762 return false; 763 unsigned Reg = VA.getLocReg(); 764 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 765 return false; 766 } 767 return true; 768} 769 770SDValue 771SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 772 SmallVectorImpl<SDValue> &InVals) const { 773 SelectionDAG &DAG = CLI.DAG; 774 SDLoc &DL = CLI.DL; 775 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 776 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 777 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 778 SDValue Chain = CLI.Chain; 779 SDValue Callee = CLI.Callee; 780 bool &IsTailCall = CLI.IsTailCall; 781 CallingConv::ID CallConv = CLI.CallConv; 782 bool IsVarArg = CLI.IsVarArg; 783 MachineFunction &MF = DAG.getMachineFunction(); 784 EVT PtrVT = getPointerTy(); 785 786 // Analyze the operands of the call, assigning locations to each operand. 787 SmallVector<CCValAssign, 16> ArgLocs; 788 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 789 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 790 791 // We don't support GuaranteedTailCallOpt, only automatically-detected 792 // sibling calls. 793 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 794 IsTailCall = false; 795 796 // Get a count of how many bytes are to be pushed on the stack. 797 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 798 799 // Mark the start of the call. 800 if (!IsTailCall) 801 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 802 DL); 803 804 // Copy argument values to their designated locations. 805 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 806 SmallVector<SDValue, 8> MemOpChains; 807 SDValue StackPtr; 808 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 809 CCValAssign &VA = ArgLocs[I]; 810 SDValue ArgValue = OutVals[I]; 811 812 if (VA.getLocInfo() == CCValAssign::Indirect) { 813 // Store the argument in a stack slot and pass its address. 814 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 815 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 816 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 817 MachinePointerInfo::getFixedStack(FI), 818 false, false, 0)); 819 ArgValue = SpillSlot; 820 } else 821 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 822 823 if (VA.isRegLoc()) 824 // Queue up the argument copies and emit them at the end. 825 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 826 else { 827 assert(VA.isMemLoc() && "Argument not register or memory"); 828 829 // Work out the address of the stack slot. Unpromoted ints and 830 // floats are passed as right-justified 8-byte values. 831 if (!StackPtr.getNode()) 832 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 833 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 834 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 835 Offset += 4; 836 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 837 DAG.getIntPtrConstant(Offset)); 838 839 // Emit the store. 840 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 841 MachinePointerInfo(), 842 false, false, 0)); 843 } 844 } 845 846 // Join the stores, which are independent of one another. 847 if (!MemOpChains.empty()) 848 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 849 &MemOpChains[0], MemOpChains.size()); 850 851 // Accept direct calls by converting symbolic call addresses to the 852 // associated Target* opcodes. Force %r1 to be used for indirect 853 // tail calls. 854 SDValue Glue; 855 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 856 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 857 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 858 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 859 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 860 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 861 } else if (IsTailCall) { 862 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 863 Glue = Chain.getValue(1); 864 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 865 } 866 867 // Build a sequence of copy-to-reg nodes, chained and glued together. 868 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 869 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 870 RegsToPass[I].second, Glue); 871 Glue = Chain.getValue(1); 872 } 873 874 // The first call operand is the chain and the second is the target address. 875 SmallVector<SDValue, 8> Ops; 876 Ops.push_back(Chain); 877 Ops.push_back(Callee); 878 879 // Add argument registers to the end of the list so that they are 880 // known live into the call. 881 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 882 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 883 RegsToPass[I].second.getValueType())); 884 885 // Glue the call to the argument copies, if any. 886 if (Glue.getNode()) 887 Ops.push_back(Glue); 888 889 // Emit the call. 890 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 891 if (IsTailCall) 892 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, &Ops[0], Ops.size()); 893 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 894 Glue = Chain.getValue(1); 895 896 // Mark the end of the call, which is glued to the call itself. 897 Chain = DAG.getCALLSEQ_END(Chain, 898 DAG.getConstant(NumBytes, PtrVT, true), 899 DAG.getConstant(0, PtrVT, true), 900 Glue, DL); 901 Glue = Chain.getValue(1); 902 903 // Assign locations to each value returned by this call. 904 SmallVector<CCValAssign, 16> RetLocs; 905 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 906 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 907 908 // Copy all of the result registers out of their specified physreg. 909 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 910 CCValAssign &VA = RetLocs[I]; 911 912 // Copy the value out, gluing the copy to the end of the call sequence. 913 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 914 VA.getLocVT(), Glue); 915 Chain = RetValue.getValue(1); 916 Glue = RetValue.getValue(2); 917 918 // Convert the value of the return register into the value that's 919 // being returned. 920 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 921 } 922 923 return Chain; 924} 925 926SDValue 927SystemZTargetLowering::LowerReturn(SDValue Chain, 928 CallingConv::ID CallConv, bool IsVarArg, 929 const SmallVectorImpl<ISD::OutputArg> &Outs, 930 const SmallVectorImpl<SDValue> &OutVals, 931 SDLoc DL, SelectionDAG &DAG) const { 932 MachineFunction &MF = DAG.getMachineFunction(); 933 934 // Assign locations to each returned value. 935 SmallVector<CCValAssign, 16> RetLocs; 936 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 937 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 938 939 // Quick exit for void returns 940 if (RetLocs.empty()) 941 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 942 943 // Copy the result values into the output registers. 944 SDValue Glue; 945 SmallVector<SDValue, 4> RetOps; 946 RetOps.push_back(Chain); 947 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 948 CCValAssign &VA = RetLocs[I]; 949 SDValue RetValue = OutVals[I]; 950 951 // Make the return register live on exit. 952 assert(VA.isRegLoc() && "Can only return in registers!"); 953 954 // Promote the value as required. 955 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 956 957 // Chain and glue the copies together. 958 unsigned Reg = VA.getLocReg(); 959 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 960 Glue = Chain.getValue(1); 961 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 962 } 963 964 // Update chain and glue. 965 RetOps[0] = Chain; 966 if (Glue.getNode()) 967 RetOps.push_back(Glue); 968 969 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 970 RetOps.data(), RetOps.size()); 971} 972 973// CC is a comparison that will be implemented using an integer or 974// floating-point comparison. Return the condition code mask for 975// a branch on true. In the integer case, CCMASK_CMP_UO is set for 976// unsigned comparisons and clear for signed ones. In the floating-point 977// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 978static unsigned CCMaskForCondCode(ISD::CondCode CC) { 979#define CONV(X) \ 980 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 981 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 982 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 983 984 switch (CC) { 985 default: 986 llvm_unreachable("Invalid integer condition!"); 987 988 CONV(EQ); 989 CONV(NE); 990 CONV(GT); 991 CONV(GE); 992 CONV(LT); 993 CONV(LE); 994 995 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 996 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 997 } 998#undef CONV 999} 1000 1001// Return a sequence for getting a 1 from an IPM result when CC has a 1002// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1003// The handling of CC values outside CCValid doesn't matter. 1004static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1005 // Deal with cases where the result can be taken directly from a bit 1006 // of the IPM result. 1007 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1008 return IPMConversion(0, 0, SystemZ::IPM_CC); 1009 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1010 return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1011 1012 // Deal with cases where we can add a value to force the sign bit 1013 // to contain the right value. Putting the bit in 31 means we can 1014 // use SRL rather than RISBG(L), and also makes it easier to get a 1015 // 0/-1 value, so it has priority over the other tests below. 1016 // 1017 // These sequences rely on the fact that the upper two bits of the 1018 // IPM result are zero. 1019 uint64_t TopBit = uint64_t(1) << 31; 1020 if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1021 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1022 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1023 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1024 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1025 | SystemZ::CCMASK_1 1026 | SystemZ::CCMASK_2))) 1027 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1028 if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1029 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1030 if (CCMask == (CCValid & (SystemZ::CCMASK_1 1031 | SystemZ::CCMASK_2 1032 | SystemZ::CCMASK_3))) 1033 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1034 1035 // Next try inverting the value and testing a bit. 0/1 could be 1036 // handled this way too, but we dealt with that case above. 1037 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1038 return IPMConversion(-1, 0, SystemZ::IPM_CC); 1039 1040 // Handle cases where adding a value forces a non-sign bit to contain 1041 // the right value. 1042 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1043 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1044 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1045 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1046 1047 // The remaing cases are 1, 2, 0/1/3 and 0/2/3. All these are 1048 // can be done by inverting the low CC bit and applying one of the 1049 // sign-based extractions above. 1050 if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1051 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1052 if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1053 return IPMConversion(1 << SystemZ::IPM_CC, 1054 TopBit - (3 << SystemZ::IPM_CC), 31); 1055 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1056 | SystemZ::CCMASK_1 1057 | SystemZ::CCMASK_3))) 1058 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1059 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1060 | SystemZ::CCMASK_2 1061 | SystemZ::CCMASK_3))) 1062 return IPMConversion(1 << SystemZ::IPM_CC, 1063 TopBit - (1 << SystemZ::IPM_CC), 31); 1064 1065 llvm_unreachable("Unexpected CC combination"); 1066} 1067 1068// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 1069// can be converted to a comparison against zero, adjust the operands 1070// as necessary. 1071static void adjustZeroCmp(SelectionDAG &DAG, bool &IsUnsigned, 1072 SDValue &CmpOp0, SDValue &CmpOp1, 1073 unsigned &CCMask) { 1074 if (IsUnsigned) 1075 return; 1076 1077 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(CmpOp1.getNode()); 1078 if (!ConstOp1) 1079 return; 1080 1081 int64_t Value = ConstOp1->getSExtValue(); 1082 if ((Value == -1 && CCMask == SystemZ::CCMASK_CMP_GT) || 1083 (Value == -1 && CCMask == SystemZ::CCMASK_CMP_LE) || 1084 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_LT) || 1085 (Value == 1 && CCMask == SystemZ::CCMASK_CMP_GE)) { 1086 CCMask ^= SystemZ::CCMASK_CMP_EQ; 1087 CmpOp1 = DAG.getConstant(0, CmpOp1.getValueType()); 1088 } 1089} 1090 1091// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 1092// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 1093static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 1094 SDValue &CmpOp0, SDValue &CmpOp1, 1095 unsigned &CCMask) { 1096 // For us to make any changes, it must a comparison between a single-use 1097 // load and a constant. 1098 if (!CmpOp0.hasOneUse() || 1099 CmpOp0.getOpcode() != ISD::LOAD || 1100 CmpOp1.getOpcode() != ISD::Constant) 1101 return; 1102 1103 // We must have an 8- or 16-bit load. 1104 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 1105 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1106 if (NumBits != 8 && NumBits != 16) 1107 return; 1108 1109 // The load must be an extending one and the constant must be within the 1110 // range of the unextended value. 1111 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 1112 uint64_t Value = Constant->getZExtValue(); 1113 uint64_t Mask = (1 << NumBits) - 1; 1114 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1115 int64_t SignedValue = Constant->getSExtValue(); 1116 if (uint64_t(SignedValue) + (1ULL << (NumBits - 1)) > Mask) 1117 return; 1118 // Unsigned comparison between two sign-extended values is equivalent 1119 // to unsigned comparison between two zero-extended values. 1120 if (IsUnsigned) 1121 Value &= Mask; 1122 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 1123 CCMask == SystemZ::CCMASK_CMP_NE) 1124 // Any choice of IsUnsigned is OK for equality comparisons. 1125 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 1126 // but since we use CLHHSI for zero extensions, it seems better 1127 // to be consistent and do the same here. 1128 Value &= Mask, IsUnsigned = true; 1129 else if (NumBits == 8) { 1130 // Try to treat the comparison as unsigned, so that we can use CLI. 1131 // Adjust CCMask and Value as necessary. 1132 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 1133 // Test whether the high bit of the byte is set. 1134 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 1135 else if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_GE) 1136 // Test whether the high bit of the byte is clear. 1137 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 1138 else 1139 // No instruction exists for this combination. 1140 return; 1141 } 1142 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1143 if (Value > Mask) 1144 return; 1145 // Signed comparison between two zero-extended values is equivalent 1146 // to unsigned comparison. 1147 IsUnsigned = true; 1148 } else 1149 return; 1150 1151 // Make sure that the first operand is an i32 of the right extension type. 1152 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 1153 if (CmpOp0.getValueType() != MVT::i32 || 1154 Load->getExtensionType() != ExtType) 1155 CmpOp0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1156 Load->getChain(), Load->getBasePtr(), 1157 Load->getPointerInfo(), Load->getMemoryVT(), 1158 Load->isVolatile(), Load->isNonTemporal(), 1159 Load->getAlignment()); 1160 1161 // Make sure that the second operand is an i32 with the right value. 1162 if (CmpOp1.getValueType() != MVT::i32 || 1163 Value != Constant->getZExtValue()) 1164 CmpOp1 = DAG.getConstant(Value, MVT::i32); 1165} 1166 1167// Return true if Op is either an unextended load, or a load suitable 1168// for integer register-memory comparisons of type ICmpType. 1169static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 1170 LoadSDNode *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1171 if (Load) { 1172 // There are no instructions to compare a register with a memory byte. 1173 if (Load->getMemoryVT() == MVT::i8) 1174 return false; 1175 // Otherwise decide on extension type. 1176 switch (Load->getExtensionType()) { 1177 case ISD::NON_EXTLOAD: 1178 return true; 1179 case ISD::SEXTLOAD: 1180 return ICmpType != SystemZICMP::UnsignedOnly; 1181 case ISD::ZEXTLOAD: 1182 return ICmpType != SystemZICMP::SignedOnly; 1183 default: 1184 break; 1185 } 1186 } 1187 return false; 1188} 1189 1190// Return true if it is better to swap comparison operands Op0 and Op1. 1191// ICmpType is the type of an integer comparison. 1192static bool shouldSwapCmpOperands(SDValue Op0, SDValue Op1, 1193 unsigned ICmpType) { 1194 // Leave f128 comparisons alone, since they have no memory forms. 1195 if (Op0.getValueType() == MVT::f128) 1196 return false; 1197 1198 // Always keep a floating-point constant second, since comparisons with 1199 // zero can use LOAD TEST and comparisons with other constants make a 1200 // natural memory operand. 1201 if (isa<ConstantFPSDNode>(Op1)) 1202 return false; 1203 1204 // Never swap comparisons with zero since there are many ways to optimize 1205 // those later. 1206 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1); 1207 if (COp1 && COp1->getZExtValue() == 0) 1208 return false; 1209 1210 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1211 // In that case we generally prefer the memory to be second. 1212 if ((isNaturalMemoryOperand(Op0, ICmpType) && Op0.hasOneUse()) && 1213 !(isNaturalMemoryOperand(Op1, ICmpType) && Op1.hasOneUse())) { 1214 // The only exceptions are when the second operand is a constant and 1215 // we can use things like CHHSI. 1216 if (!COp1) 1217 return true; 1218 // The unsigned memory-immediate instructions can handle 16-bit 1219 // unsigned integers. 1220 if (ICmpType != SystemZICMP::SignedOnly && 1221 isUInt<16>(COp1->getZExtValue())) 1222 return false; 1223 // The signed memory-immediate instructions can handle 16-bit 1224 // signed integers. 1225 if (ICmpType != SystemZICMP::UnsignedOnly && 1226 isInt<16>(COp1->getSExtValue())) 1227 return false; 1228 return true; 1229 } 1230 return false; 1231} 1232 1233// Return true if shift operation N has an in-range constant shift value. 1234// Store it in ShiftVal if so. 1235static bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 1236 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1237 if (!Shift) 1238 return false; 1239 1240 uint64_t Amount = Shift->getZExtValue(); 1241 if (Amount >= N.getValueType().getSizeInBits()) 1242 return false; 1243 1244 ShiftVal = Amount; 1245 return true; 1246} 1247 1248// Check whether an AND with Mask is suitable for a TEST UNDER MASK 1249// instruction and whether the CC value is descriptive enough to handle 1250// a comparison of type Opcode between the AND result and CmpVal. 1251// CCMask says which comparison result is being tested and BitSize is 1252// the number of bits in the operands. If TEST UNDER MASK can be used, 1253// return the corresponding CC mask, otherwise return 0. 1254static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 1255 uint64_t Mask, uint64_t CmpVal, 1256 unsigned ICmpType) { 1257 assert(Mask != 0 && "ANDs with zero should have been removed by now"); 1258 1259 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 1260 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 1261 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 1262 return 0; 1263 1264 // Work out the masks for the lowest and highest bits. 1265 unsigned HighShift = 63 - countLeadingZeros(Mask); 1266 uint64_t High = uint64_t(1) << HighShift; 1267 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 1268 1269 // Signed ordered comparisons are effectively unsigned if the sign 1270 // bit is dropped. 1271 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 1272 1273 // Check for equality comparisons with 0, or the equivalent. 1274 if (CmpVal == 0) { 1275 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1276 return SystemZ::CCMASK_TM_ALL_0; 1277 if (CCMask == SystemZ::CCMASK_CMP_NE) 1278 return SystemZ::CCMASK_TM_SOME_1; 1279 } 1280 if (EffectivelyUnsigned && CmpVal <= Low) { 1281 if (CCMask == SystemZ::CCMASK_CMP_LT) 1282 return SystemZ::CCMASK_TM_ALL_0; 1283 if (CCMask == SystemZ::CCMASK_CMP_GE) 1284 return SystemZ::CCMASK_TM_SOME_1; 1285 } 1286 if (EffectivelyUnsigned && CmpVal < Low) { 1287 if (CCMask == SystemZ::CCMASK_CMP_LE) 1288 return SystemZ::CCMASK_TM_ALL_0; 1289 if (CCMask == SystemZ::CCMASK_CMP_GT) 1290 return SystemZ::CCMASK_TM_SOME_1; 1291 } 1292 1293 // Check for equality comparisons with the mask, or the equivalent. 1294 if (CmpVal == Mask) { 1295 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1296 return SystemZ::CCMASK_TM_ALL_1; 1297 if (CCMask == SystemZ::CCMASK_CMP_NE) 1298 return SystemZ::CCMASK_TM_SOME_0; 1299 } 1300 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 1301 if (CCMask == SystemZ::CCMASK_CMP_GT) 1302 return SystemZ::CCMASK_TM_ALL_1; 1303 if (CCMask == SystemZ::CCMASK_CMP_LE) 1304 return SystemZ::CCMASK_TM_SOME_0; 1305 } 1306 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 1307 if (CCMask == SystemZ::CCMASK_CMP_GE) 1308 return SystemZ::CCMASK_TM_ALL_1; 1309 if (CCMask == SystemZ::CCMASK_CMP_LT) 1310 return SystemZ::CCMASK_TM_SOME_0; 1311 } 1312 1313 // Check for ordered comparisons with the top bit. 1314 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 1315 if (CCMask == SystemZ::CCMASK_CMP_LE) 1316 return SystemZ::CCMASK_TM_MSB_0; 1317 if (CCMask == SystemZ::CCMASK_CMP_GT) 1318 return SystemZ::CCMASK_TM_MSB_1; 1319 } 1320 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 1321 if (CCMask == SystemZ::CCMASK_CMP_LT) 1322 return SystemZ::CCMASK_TM_MSB_0; 1323 if (CCMask == SystemZ::CCMASK_CMP_GE) 1324 return SystemZ::CCMASK_TM_MSB_1; 1325 } 1326 1327 // If there are just two bits, we can do equality checks for Low and High 1328 // as well. 1329 if (Mask == Low + High) { 1330 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 1331 return SystemZ::CCMASK_TM_MIXED_MSB_0; 1332 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 1333 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 1334 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 1335 return SystemZ::CCMASK_TM_MIXED_MSB_1; 1336 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 1337 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 1338 } 1339 1340 // Looks like we've exhausted our options. 1341 return 0; 1342} 1343 1344// See whether the comparison (Opcode CmpOp0, CmpOp1, ICmpType) can be 1345// implemented as a TEST UNDER MASK instruction when the condition being 1346// tested is as described by CCValid and CCMask. Update the arguments 1347// with the TM version if so. 1348static void adjustForTestUnderMask(SelectionDAG &DAG, unsigned &Opcode, 1349 SDValue &CmpOp0, SDValue &CmpOp1, 1350 unsigned &CCValid, unsigned &CCMask, 1351 unsigned &ICmpType) { 1352 // Check that we have a comparison with a constant. 1353 ConstantSDNode *ConstCmpOp1 = dyn_cast<ConstantSDNode>(CmpOp1); 1354 if (!ConstCmpOp1) 1355 return; 1356 uint64_t CmpVal = ConstCmpOp1->getZExtValue(); 1357 1358 // Check whether the nonconstant input is an AND with a constant mask. 1359 if (CmpOp0.getOpcode() != ISD::AND) 1360 return; 1361 SDValue AndOp0 = CmpOp0.getOperand(0); 1362 SDValue AndOp1 = CmpOp0.getOperand(1); 1363 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(AndOp1.getNode()); 1364 if (!Mask) 1365 return; 1366 uint64_t MaskVal = Mask->getZExtValue(); 1367 1368 // Check whether the combination of mask, comparison value and comparison 1369 // type are suitable. 1370 unsigned BitSize = CmpOp0.getValueType().getSizeInBits(); 1371 unsigned NewCCMask, ShiftVal; 1372 if (ICmpType != SystemZICMP::SignedOnly && 1373 AndOp0.getOpcode() == ISD::SHL && 1374 isSimpleShift(AndOp0, ShiftVal) && 1375 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask, MaskVal >> ShiftVal, 1376 CmpVal >> ShiftVal, 1377 SystemZICMP::Any))) { 1378 AndOp0 = AndOp0.getOperand(0); 1379 AndOp1 = DAG.getConstant(MaskVal >> ShiftVal, AndOp0.getValueType()); 1380 } else if (ICmpType != SystemZICMP::SignedOnly && 1381 AndOp0.getOpcode() == ISD::SRL && 1382 isSimpleShift(AndOp0, ShiftVal) && 1383 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask, 1384 MaskVal << ShiftVal, 1385 CmpVal << ShiftVal, 1386 SystemZICMP::UnsignedOnly))) { 1387 AndOp0 = AndOp0.getOperand(0); 1388 AndOp1 = DAG.getConstant(MaskVal << ShiftVal, AndOp0.getValueType()); 1389 } else { 1390 NewCCMask = getTestUnderMaskCond(BitSize, CCMask, MaskVal, CmpVal, 1391 ICmpType); 1392 if (!NewCCMask) 1393 return; 1394 } 1395 1396 // Go ahead and make the change. 1397 Opcode = SystemZISD::TM; 1398 CmpOp0 = AndOp0; 1399 CmpOp1 = AndOp1; 1400 ICmpType = (bool(NewCCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 1401 bool(NewCCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 1402 CCValid = SystemZ::CCMASK_TM; 1403 CCMask = NewCCMask; 1404} 1405 1406// Return a target node that compares CmpOp0 with CmpOp1 and stores a 1407// 2-bit result in CC. Set CCValid to the CCMASK_* of all possible 1408// 2-bit results and CCMask to the subset of those results that are 1409// associated with Cond. 1410static SDValue emitCmp(const SystemZTargetMachine &TM, SelectionDAG &DAG, 1411 SDLoc DL, SDValue CmpOp0, SDValue CmpOp1, 1412 ISD::CondCode Cond, unsigned &CCValid, 1413 unsigned &CCMask) { 1414 bool IsUnsigned = false; 1415 CCMask = CCMaskForCondCode(Cond); 1416 unsigned Opcode, ICmpType = 0; 1417 if (CmpOp0.getValueType().isFloatingPoint()) { 1418 CCValid = SystemZ::CCMASK_FCMP; 1419 Opcode = SystemZISD::FCMP; 1420 } else { 1421 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 1422 CCValid = SystemZ::CCMASK_ICMP; 1423 CCMask &= CCValid; 1424 adjustZeroCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1425 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 1426 Opcode = SystemZISD::ICMP; 1427 // Choose the type of comparison. Equality and inequality tests can 1428 // use either signed or unsigned comparisons. The choice also doesn't 1429 // matter if both sign bits are known to be clear. In those cases we 1430 // want to give the main isel code the freedom to choose whichever 1431 // form fits best. 1432 if (CCMask == SystemZ::CCMASK_CMP_EQ || 1433 CCMask == SystemZ::CCMASK_CMP_NE || 1434 (DAG.SignBitIsZero(CmpOp0) && DAG.SignBitIsZero(CmpOp1))) 1435 ICmpType = SystemZICMP::Any; 1436 else if (IsUnsigned) 1437 ICmpType = SystemZICMP::UnsignedOnly; 1438 else 1439 ICmpType = SystemZICMP::SignedOnly; 1440 } 1441 1442 if (shouldSwapCmpOperands(CmpOp0, CmpOp1, ICmpType)) { 1443 std::swap(CmpOp0, CmpOp1); 1444 CCMask = ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1445 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1446 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1447 (CCMask & SystemZ::CCMASK_CMP_UO)); 1448 } 1449 1450 adjustForTestUnderMask(DAG, Opcode, CmpOp0, CmpOp1, CCValid, CCMask, 1451 ICmpType); 1452 if (Opcode == SystemZISD::ICMP || Opcode == SystemZISD::TM) 1453 return DAG.getNode(Opcode, DL, MVT::Glue, CmpOp0, CmpOp1, 1454 DAG.getConstant(ICmpType, MVT::i32)); 1455 return DAG.getNode(Opcode, DL, MVT::Glue, CmpOp0, CmpOp1); 1456} 1457 1458// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1459// 64 bits. Extend is the extension type to use. Store the high part 1460// in Hi and the low part in Lo. 1461static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1462 unsigned Extend, SDValue Op0, SDValue Op1, 1463 SDValue &Hi, SDValue &Lo) { 1464 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1465 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1466 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1467 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1468 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1469 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1470} 1471 1472// Lower a binary operation that produces two VT results, one in each 1473// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1474// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1475// on the extended Op0 and (unextended) Op1. Store the even register result 1476// in Even and the odd register result in Odd. 1477static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1478 unsigned Extend, unsigned Opcode, 1479 SDValue Op0, SDValue Op1, 1480 SDValue &Even, SDValue &Odd) { 1481 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1482 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1483 SDValue(In128, 0), Op1); 1484 bool Is32Bit = is32Bit(VT); 1485 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 1486 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 1487} 1488 1489SDValue SystemZTargetLowering::lowerSETCC(SDValue Op, 1490 SelectionDAG &DAG) const { 1491 SDValue CmpOp0 = Op.getOperand(0); 1492 SDValue CmpOp1 = Op.getOperand(1); 1493 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 1494 SDLoc DL(Op); 1495 1496 unsigned CCValid, CCMask; 1497 SDValue Glue = emitCmp(TM, DAG, DL, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1498 1499 IPMConversion Conversion = getIPMConversion(CCValid, CCMask); 1500 SDValue Result = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue); 1501 1502 if (Conversion.XORValue) 1503 Result = DAG.getNode(ISD::XOR, DL, MVT::i32, Result, 1504 DAG.getConstant(Conversion.XORValue, MVT::i32)); 1505 1506 if (Conversion.AddValue) 1507 Result = DAG.getNode(ISD::ADD, DL, MVT::i32, Result, 1508 DAG.getConstant(Conversion.AddValue, MVT::i32)); 1509 1510 // The SHR/AND sequence should get optimized to an RISBG. 1511 Result = DAG.getNode(ISD::SRL, DL, MVT::i32, Result, 1512 DAG.getConstant(Conversion.Bit, MVT::i32)); 1513 if (Conversion.Bit != 31) 1514 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, 1515 DAG.getConstant(1, MVT::i32)); 1516 return Result; 1517} 1518 1519SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1520 SDValue Chain = Op.getOperand(0); 1521 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1522 SDValue CmpOp0 = Op.getOperand(2); 1523 SDValue CmpOp1 = Op.getOperand(3); 1524 SDValue Dest = Op.getOperand(4); 1525 SDLoc DL(Op); 1526 1527 unsigned CCValid, CCMask; 1528 SDValue Flags = emitCmp(TM, DAG, DL, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1529 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1530 Chain, DAG.getConstant(CCValid, MVT::i32), 1531 DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 1532} 1533 1534SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1535 SelectionDAG &DAG) const { 1536 SDValue CmpOp0 = Op.getOperand(0); 1537 SDValue CmpOp1 = Op.getOperand(1); 1538 SDValue TrueOp = Op.getOperand(2); 1539 SDValue FalseOp = Op.getOperand(3); 1540 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1541 SDLoc DL(Op); 1542 1543 unsigned CCValid, CCMask; 1544 SDValue Flags = emitCmp(TM, DAG, DL, CmpOp0, CmpOp1, CC, CCValid, CCMask); 1545 1546 SmallVector<SDValue, 5> Ops; 1547 Ops.push_back(TrueOp); 1548 Ops.push_back(FalseOp); 1549 Ops.push_back(DAG.getConstant(CCValid, MVT::i32)); 1550 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1551 Ops.push_back(Flags); 1552 1553 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1554 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1555} 1556 1557SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1558 SelectionDAG &DAG) const { 1559 SDLoc DL(Node); 1560 const GlobalValue *GV = Node->getGlobal(); 1561 int64_t Offset = Node->getOffset(); 1562 EVT PtrVT = getPointerTy(); 1563 Reloc::Model RM = TM.getRelocationModel(); 1564 CodeModel::Model CM = TM.getCodeModel(); 1565 1566 SDValue Result; 1567 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1568 // Assign anchors at 1<<12 byte boundaries. 1569 uint64_t Anchor = Offset & ~uint64_t(0xfff); 1570 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 1571 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1572 1573 // The offset can be folded into the address if it is aligned to a halfword. 1574 Offset -= Anchor; 1575 if (Offset != 0 && (Offset & 1) == 0) { 1576 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 1577 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 1578 Offset = 0; 1579 } 1580 } else { 1581 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1582 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1583 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1584 MachinePointerInfo::getGOT(), false, false, false, 0); 1585 } 1586 1587 // If there was a non-zero offset that we didn't fold, create an explicit 1588 // addition for it. 1589 if (Offset != 0) 1590 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1591 DAG.getConstant(Offset, PtrVT)); 1592 1593 return Result; 1594} 1595 1596SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1597 SelectionDAG &DAG) const { 1598 SDLoc DL(Node); 1599 const GlobalValue *GV = Node->getGlobal(); 1600 EVT PtrVT = getPointerTy(); 1601 TLSModel::Model model = TM.getTLSModel(GV); 1602 1603 if (model != TLSModel::LocalExec) 1604 llvm_unreachable("only local-exec TLS mode supported"); 1605 1606 // The high part of the thread pointer is in access register 0. 1607 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1608 DAG.getConstant(0, MVT::i32)); 1609 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1610 1611 // The low part of the thread pointer is in access register 1. 1612 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1613 DAG.getConstant(1, MVT::i32)); 1614 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1615 1616 // Merge them into a single 64-bit address. 1617 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1618 DAG.getConstant(32, PtrVT)); 1619 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1620 1621 // Get the offset of GA from the thread pointer. 1622 SystemZConstantPoolValue *CPV = 1623 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1624 1625 // Force the offset into the constant pool and load it from there. 1626 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1627 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1628 CPAddr, MachinePointerInfo::getConstantPool(), 1629 false, false, false, 0); 1630 1631 // Add the base and offset together. 1632 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1633} 1634 1635SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1636 SelectionDAG &DAG) const { 1637 SDLoc DL(Node); 1638 const BlockAddress *BA = Node->getBlockAddress(); 1639 int64_t Offset = Node->getOffset(); 1640 EVT PtrVT = getPointerTy(); 1641 1642 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1643 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1644 return Result; 1645} 1646 1647SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1648 SelectionDAG &DAG) const { 1649 SDLoc DL(JT); 1650 EVT PtrVT = getPointerTy(); 1651 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1652 1653 // Use LARL to load the address of the table. 1654 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1655} 1656 1657SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1658 SelectionDAG &DAG) const { 1659 SDLoc DL(CP); 1660 EVT PtrVT = getPointerTy(); 1661 1662 SDValue Result; 1663 if (CP->isMachineConstantPoolEntry()) 1664 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1665 CP->getAlignment()); 1666 else 1667 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1668 CP->getAlignment(), CP->getOffset()); 1669 1670 // Use LARL to load the address of the constant pool entry. 1671 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1672} 1673 1674SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1675 SelectionDAG &DAG) const { 1676 SDLoc DL(Op); 1677 SDValue In = Op.getOperand(0); 1678 EVT InVT = In.getValueType(); 1679 EVT ResVT = Op.getValueType(); 1680 1681 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1682 SDValue In64; 1683 if (Subtarget.hasHighWord()) { 1684 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, 1685 MVT::i64); 1686 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1687 MVT::i64, SDValue(U64, 0), In); 1688 } else { 1689 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1690 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, 1691 DAG.getConstant(32, MVT::i64)); 1692 } 1693 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); 1694 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, 1695 DL, MVT::f32, Out64); 1696 } 1697 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1698 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1699 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1700 MVT::f64, SDValue(U64, 0), In); 1701 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 1702 if (Subtarget.hasHighWord()) 1703 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, 1704 MVT::i32, Out64); 1705 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, 1706 DAG.getConstant(32, MVT::i64)); 1707 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1708 } 1709 llvm_unreachable("Unexpected bitcast combination"); 1710} 1711 1712SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1713 SelectionDAG &DAG) const { 1714 MachineFunction &MF = DAG.getMachineFunction(); 1715 SystemZMachineFunctionInfo *FuncInfo = 1716 MF.getInfo<SystemZMachineFunctionInfo>(); 1717 EVT PtrVT = getPointerTy(); 1718 1719 SDValue Chain = Op.getOperand(0); 1720 SDValue Addr = Op.getOperand(1); 1721 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1722 SDLoc DL(Op); 1723 1724 // The initial values of each field. 1725 const unsigned NumFields = 4; 1726 SDValue Fields[NumFields] = { 1727 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1728 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1729 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1730 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1731 }; 1732 1733 // Store each field into its respective slot. 1734 SDValue MemOps[NumFields]; 1735 unsigned Offset = 0; 1736 for (unsigned I = 0; I < NumFields; ++I) { 1737 SDValue FieldAddr = Addr; 1738 if (Offset != 0) 1739 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1740 DAG.getIntPtrConstant(Offset)); 1741 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1742 MachinePointerInfo(SV, Offset), 1743 false, false, 0); 1744 Offset += 8; 1745 } 1746 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1747} 1748 1749SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1750 SelectionDAG &DAG) const { 1751 SDValue Chain = Op.getOperand(0); 1752 SDValue DstPtr = Op.getOperand(1); 1753 SDValue SrcPtr = Op.getOperand(2); 1754 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1755 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1756 SDLoc DL(Op); 1757 1758 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1759 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1760 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1761} 1762 1763SDValue SystemZTargetLowering:: 1764lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1765 SDValue Chain = Op.getOperand(0); 1766 SDValue Size = Op.getOperand(1); 1767 SDLoc DL(Op); 1768 1769 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1770 1771 // Get a reference to the stack pointer. 1772 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1773 1774 // Get the new stack pointer value. 1775 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1776 1777 // Copy the new stack pointer back. 1778 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1779 1780 // The allocated data lives above the 160 bytes allocated for the standard 1781 // frame, plus any outgoing stack arguments. We don't know how much that 1782 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1783 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1784 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1785 1786 SDValue Ops[2] = { Result, Chain }; 1787 return DAG.getMergeValues(Ops, 2, DL); 1788} 1789 1790SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 1791 SelectionDAG &DAG) const { 1792 EVT VT = Op.getValueType(); 1793 SDLoc DL(Op); 1794 SDValue Ops[2]; 1795 if (is32Bit(VT)) 1796 // Just do a normal 64-bit multiplication and extract the results. 1797 // We define this so that it can be used for constant division. 1798 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 1799 Op.getOperand(1), Ops[1], Ops[0]); 1800 else { 1801 // Do a full 128-bit multiplication based on UMUL_LOHI64: 1802 // 1803 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 1804 // 1805 // but using the fact that the upper halves are either all zeros 1806 // or all ones: 1807 // 1808 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 1809 // 1810 // and grouping the right terms together since they are quicker than the 1811 // multiplication: 1812 // 1813 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 1814 SDValue C63 = DAG.getConstant(63, MVT::i64); 1815 SDValue LL = Op.getOperand(0); 1816 SDValue RL = Op.getOperand(1); 1817 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 1818 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 1819 // UMUL_LOHI64 returns the low result in the odd register and the high 1820 // result in the even register. SMUL_LOHI is defined to return the 1821 // low half first, so the results are in reverse order. 1822 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1823 LL, RL, Ops[1], Ops[0]); 1824 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 1825 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 1826 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 1827 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 1828 } 1829 return DAG.getMergeValues(Ops, 2, DL); 1830} 1831 1832SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1833 SelectionDAG &DAG) const { 1834 EVT VT = Op.getValueType(); 1835 SDLoc DL(Op); 1836 SDValue Ops[2]; 1837 if (is32Bit(VT)) 1838 // Just do a normal 64-bit multiplication and extract the results. 1839 // We define this so that it can be used for constant division. 1840 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 1841 Op.getOperand(1), Ops[1], Ops[0]); 1842 else 1843 // UMUL_LOHI64 returns the low result in the odd register and the high 1844 // result in the even register. UMUL_LOHI is defined to return the 1845 // low half first, so the results are in reverse order. 1846 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1847 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1848 return DAG.getMergeValues(Ops, 2, DL); 1849} 1850 1851SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1852 SelectionDAG &DAG) const { 1853 SDValue Op0 = Op.getOperand(0); 1854 SDValue Op1 = Op.getOperand(1); 1855 EVT VT = Op.getValueType(); 1856 SDLoc DL(Op); 1857 unsigned Opcode; 1858 1859 // We use DSGF for 32-bit division. 1860 if (is32Bit(VT)) { 1861 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1862 Opcode = SystemZISD::SDIVREM32; 1863 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 1864 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 1865 Opcode = SystemZISD::SDIVREM32; 1866 } else 1867 Opcode = SystemZISD::SDIVREM64; 1868 1869 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1870 // input is "don't care". The instruction returns the remainder in 1871 // the even register and the quotient in the odd register. 1872 SDValue Ops[2]; 1873 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 1874 Op0, Op1, Ops[1], Ops[0]); 1875 return DAG.getMergeValues(Ops, 2, DL); 1876} 1877 1878SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1879 SelectionDAG &DAG) const { 1880 EVT VT = Op.getValueType(); 1881 SDLoc DL(Op); 1882 1883 // DL(G) uses a double-width dividend, so we need to clear the even 1884 // register in the GR128 input. The instruction returns the remainder 1885 // in the even register and the quotient in the odd register. 1886 SDValue Ops[2]; 1887 if (is32Bit(VT)) 1888 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1889 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1890 else 1891 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1892 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1893 return DAG.getMergeValues(Ops, 2, DL); 1894} 1895 1896SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1897 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1898 1899 // Get the known-zero masks for each operand. 1900 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1901 APInt KnownZero[2], KnownOne[2]; 1902 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1903 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1904 1905 // See if the upper 32 bits of one operand and the lower 32 bits of the 1906 // other are known zero. They are the low and high operands respectively. 1907 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1908 KnownZero[1].getZExtValue() }; 1909 unsigned High, Low; 1910 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1911 High = 1, Low = 0; 1912 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1913 High = 0, Low = 1; 1914 else 1915 return Op; 1916 1917 SDValue LowOp = Ops[Low]; 1918 SDValue HighOp = Ops[High]; 1919 1920 // If the high part is a constant, we're better off using IILH. 1921 if (HighOp.getOpcode() == ISD::Constant) 1922 return Op; 1923 1924 // If the low part is a constant that is outside the range of LHI, 1925 // then we're better off using IILF. 1926 if (LowOp.getOpcode() == ISD::Constant) { 1927 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1928 if (!isInt<16>(Value)) 1929 return Op; 1930 } 1931 1932 // Check whether the high part is an AND that doesn't change the 1933 // high 32 bits and just masks out low bits. We can skip it if so. 1934 if (HighOp.getOpcode() == ISD::AND && 1935 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1936 SDValue HighOp0 = HighOp.getOperand(0); 1937 uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue(); 1938 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff)))) 1939 HighOp = HighOp0; 1940 } 1941 1942 // Take advantage of the fact that all GR32 operations only change the 1943 // low 32 bits by truncating Low to an i32 and inserting it directly 1944 // using a subreg. The interesting cases are those where the truncation 1945 // can be folded. 1946 SDLoc DL(Op); 1947 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1948 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, 1949 MVT::i64, HighOp, Low32); 1950} 1951 1952// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1953// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1954SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1955 SelectionDAG &DAG, 1956 unsigned Opcode) const { 1957 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1958 1959 // 32-bit operations need no code outside the main loop. 1960 EVT NarrowVT = Node->getMemoryVT(); 1961 EVT WideVT = MVT::i32; 1962 if (NarrowVT == WideVT) 1963 return Op; 1964 1965 int64_t BitSize = NarrowVT.getSizeInBits(); 1966 SDValue ChainIn = Node->getChain(); 1967 SDValue Addr = Node->getBasePtr(); 1968 SDValue Src2 = Node->getVal(); 1969 MachineMemOperand *MMO = Node->getMemOperand(); 1970 SDLoc DL(Node); 1971 EVT PtrVT = Addr.getValueType(); 1972 1973 // Convert atomic subtracts of constants into additions. 1974 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1975 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1976 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1977 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1978 } 1979 1980 // Get the address of the containing word. 1981 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1982 DAG.getConstant(-4, PtrVT)); 1983 1984 // Get the number of bits that the word must be rotated left in order 1985 // to bring the field to the top bits of a GR32. 1986 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1987 DAG.getConstant(3, PtrVT)); 1988 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1989 1990 // Get the complementing shift amount, for rotating a field in the top 1991 // bits back to its proper position. 1992 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1993 DAG.getConstant(0, WideVT), BitShift); 1994 1995 // Extend the source operand to 32 bits and prepare it for the inner loop. 1996 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1997 // operations require the source to be shifted in advance. (This shift 1998 // can be folded if the source is constant.) For AND and NAND, the lower 1999 // bits must be set, while for other opcodes they should be left clear. 2000 if (Opcode != SystemZISD::ATOMIC_SWAPW) 2001 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 2002 DAG.getConstant(32 - BitSize, WideVT)); 2003 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 2004 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 2005 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 2006 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 2007 2008 // Construct the ATOMIC_LOADW_* node. 2009 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2010 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 2011 DAG.getConstant(BitSize, WideVT) }; 2012 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 2013 array_lengthof(Ops), 2014 NarrowVT, MMO); 2015 2016 // Rotate the result of the final CS so that the field is in the lower 2017 // bits of a GR32, then truncate it. 2018 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 2019 DAG.getConstant(BitSize, WideVT)); 2020 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 2021 2022 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 2023 return DAG.getMergeValues(RetOps, 2, DL); 2024} 2025 2026// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 2027// into a fullword ATOMIC_CMP_SWAPW operation. 2028SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 2029 SelectionDAG &DAG) const { 2030 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 2031 2032 // We have native support for 32-bit compare and swap. 2033 EVT NarrowVT = Node->getMemoryVT(); 2034 EVT WideVT = MVT::i32; 2035 if (NarrowVT == WideVT) 2036 return Op; 2037 2038 int64_t BitSize = NarrowVT.getSizeInBits(); 2039 SDValue ChainIn = Node->getOperand(0); 2040 SDValue Addr = Node->getOperand(1); 2041 SDValue CmpVal = Node->getOperand(2); 2042 SDValue SwapVal = Node->getOperand(3); 2043 MachineMemOperand *MMO = Node->getMemOperand(); 2044 SDLoc DL(Node); 2045 EVT PtrVT = Addr.getValueType(); 2046 2047 // Get the address of the containing word. 2048 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 2049 DAG.getConstant(-4, PtrVT)); 2050 2051 // Get the number of bits that the word must be rotated left in order 2052 // to bring the field to the top bits of a GR32. 2053 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 2054 DAG.getConstant(3, PtrVT)); 2055 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 2056 2057 // Get the complementing shift amount, for rotating a field in the top 2058 // bits back to its proper position. 2059 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 2060 DAG.getConstant(0, WideVT), BitShift); 2061 2062 // Construct the ATOMIC_CMP_SWAPW node. 2063 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2064 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 2065 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 2066 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 2067 VTList, Ops, array_lengthof(Ops), 2068 NarrowVT, MMO); 2069 return AtomicOp; 2070} 2071 2072SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 2073 SelectionDAG &DAG) const { 2074 MachineFunction &MF = DAG.getMachineFunction(); 2075 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2076 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 2077 SystemZ::R15D, Op.getValueType()); 2078} 2079 2080SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 2081 SelectionDAG &DAG) const { 2082 MachineFunction &MF = DAG.getMachineFunction(); 2083 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2084 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 2085 SystemZ::R15D, Op.getOperand(1)); 2086} 2087 2088SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op, 2089 SelectionDAG &DAG) const { 2090 bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 2091 if (!IsData) 2092 // Just preserve the chain. 2093 return Op.getOperand(0); 2094 2095 bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 2096 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ; 2097 MemIntrinsicSDNode *Node = cast<MemIntrinsicSDNode>(Op.getNode()); 2098 SDValue Ops[] = { 2099 Op.getOperand(0), 2100 DAG.getConstant(Code, MVT::i32), 2101 Op.getOperand(1) 2102 }; 2103 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, SDLoc(Op), 2104 Node->getVTList(), Ops, array_lengthof(Ops), 2105 Node->getMemoryVT(), Node->getMemOperand()); 2106} 2107 2108SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 2109 SelectionDAG &DAG) const { 2110 switch (Op.getOpcode()) { 2111 case ISD::BR_CC: 2112 return lowerBR_CC(Op, DAG); 2113 case ISD::SELECT_CC: 2114 return lowerSELECT_CC(Op, DAG); 2115 case ISD::SETCC: 2116 return lowerSETCC(Op, DAG); 2117 case ISD::GlobalAddress: 2118 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 2119 case ISD::GlobalTLSAddress: 2120 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 2121 case ISD::BlockAddress: 2122 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 2123 case ISD::JumpTable: 2124 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 2125 case ISD::ConstantPool: 2126 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 2127 case ISD::BITCAST: 2128 return lowerBITCAST(Op, DAG); 2129 case ISD::VASTART: 2130 return lowerVASTART(Op, DAG); 2131 case ISD::VACOPY: 2132 return lowerVACOPY(Op, DAG); 2133 case ISD::DYNAMIC_STACKALLOC: 2134 return lowerDYNAMIC_STACKALLOC(Op, DAG); 2135 case ISD::SMUL_LOHI: 2136 return lowerSMUL_LOHI(Op, DAG); 2137 case ISD::UMUL_LOHI: 2138 return lowerUMUL_LOHI(Op, DAG); 2139 case ISD::SDIVREM: 2140 return lowerSDIVREM(Op, DAG); 2141 case ISD::UDIVREM: 2142 return lowerUDIVREM(Op, DAG); 2143 case ISD::OR: 2144 return lowerOR(Op, DAG); 2145 case ISD::ATOMIC_SWAP: 2146 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 2147 case ISD::ATOMIC_LOAD_ADD: 2148 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 2149 case ISD::ATOMIC_LOAD_SUB: 2150 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 2151 case ISD::ATOMIC_LOAD_AND: 2152 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 2153 case ISD::ATOMIC_LOAD_OR: 2154 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 2155 case ISD::ATOMIC_LOAD_XOR: 2156 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 2157 case ISD::ATOMIC_LOAD_NAND: 2158 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 2159 case ISD::ATOMIC_LOAD_MIN: 2160 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 2161 case ISD::ATOMIC_LOAD_MAX: 2162 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 2163 case ISD::ATOMIC_LOAD_UMIN: 2164 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 2165 case ISD::ATOMIC_LOAD_UMAX: 2166 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 2167 case ISD::ATOMIC_CMP_SWAP: 2168 return lowerATOMIC_CMP_SWAP(Op, DAG); 2169 case ISD::STACKSAVE: 2170 return lowerSTACKSAVE(Op, DAG); 2171 case ISD::STACKRESTORE: 2172 return lowerSTACKRESTORE(Op, DAG); 2173 case ISD::PREFETCH: 2174 return lowerPREFETCH(Op, DAG); 2175 default: 2176 llvm_unreachable("Unexpected node to lower"); 2177 } 2178} 2179 2180const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 2181#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 2182 switch (Opcode) { 2183 OPCODE(RET_FLAG); 2184 OPCODE(CALL); 2185 OPCODE(SIBCALL); 2186 OPCODE(PCREL_WRAPPER); 2187 OPCODE(PCREL_OFFSET); 2188 OPCODE(ICMP); 2189 OPCODE(FCMP); 2190 OPCODE(TM); 2191 OPCODE(BR_CCMASK); 2192 OPCODE(SELECT_CCMASK); 2193 OPCODE(ADJDYNALLOC); 2194 OPCODE(EXTRACT_ACCESS); 2195 OPCODE(UMUL_LOHI64); 2196 OPCODE(SDIVREM64); 2197 OPCODE(UDIVREM32); 2198 OPCODE(UDIVREM64); 2199 OPCODE(MVC); 2200 OPCODE(MVC_LOOP); 2201 OPCODE(NC); 2202 OPCODE(NC_LOOP); 2203 OPCODE(OC); 2204 OPCODE(OC_LOOP); 2205 OPCODE(XC); 2206 OPCODE(XC_LOOP); 2207 OPCODE(CLC); 2208 OPCODE(CLC_LOOP); 2209 OPCODE(STRCMP); 2210 OPCODE(STPCPY); 2211 OPCODE(SEARCH_STRING); 2212 OPCODE(IPM); 2213 OPCODE(ATOMIC_SWAPW); 2214 OPCODE(ATOMIC_LOADW_ADD); 2215 OPCODE(ATOMIC_LOADW_SUB); 2216 OPCODE(ATOMIC_LOADW_AND); 2217 OPCODE(ATOMIC_LOADW_OR); 2218 OPCODE(ATOMIC_LOADW_XOR); 2219 OPCODE(ATOMIC_LOADW_NAND); 2220 OPCODE(ATOMIC_LOADW_MIN); 2221 OPCODE(ATOMIC_LOADW_MAX); 2222 OPCODE(ATOMIC_LOADW_UMIN); 2223 OPCODE(ATOMIC_LOADW_UMAX); 2224 OPCODE(ATOMIC_CMP_SWAPW); 2225 OPCODE(PREFETCH); 2226 } 2227 return NULL; 2228#undef OPCODE 2229} 2230 2231//===----------------------------------------------------------------------===// 2232// Custom insertion 2233//===----------------------------------------------------------------------===// 2234 2235// Create a new basic block after MBB. 2236static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 2237 MachineFunction &MF = *MBB->getParent(); 2238 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 2239 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 2240 return NewMBB; 2241} 2242 2243// Split MBB after MI and return the new block (the one that contains 2244// instructions after MI). 2245static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 2246 MachineBasicBlock *MBB) { 2247 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2248 NewMBB->splice(NewMBB->begin(), MBB, 2249 llvm::next(MachineBasicBlock::iterator(MI)), 2250 MBB->end()); 2251 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2252 return NewMBB; 2253} 2254 2255// Split MBB before MI and return the new block (the one that contains MI). 2256static MachineBasicBlock *splitBlockBefore(MachineInstr *MI, 2257 MachineBasicBlock *MBB) { 2258 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2259 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 2260 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2261 return NewMBB; 2262} 2263 2264// Force base value Base into a register before MI. Return the register. 2265static unsigned forceReg(MachineInstr *MI, MachineOperand &Base, 2266 const SystemZInstrInfo *TII) { 2267 if (Base.isReg()) 2268 return Base.getReg(); 2269 2270 MachineBasicBlock *MBB = MI->getParent(); 2271 MachineFunction &MF = *MBB->getParent(); 2272 MachineRegisterInfo &MRI = MF.getRegInfo(); 2273 2274 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2275 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LA), Reg) 2276 .addOperand(Base).addImm(0).addReg(0); 2277 return Reg; 2278} 2279 2280// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 2281MachineBasicBlock * 2282SystemZTargetLowering::emitSelect(MachineInstr *MI, 2283 MachineBasicBlock *MBB) const { 2284 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2285 2286 unsigned DestReg = MI->getOperand(0).getReg(); 2287 unsigned TrueReg = MI->getOperand(1).getReg(); 2288 unsigned FalseReg = MI->getOperand(2).getReg(); 2289 unsigned CCValid = MI->getOperand(3).getImm(); 2290 unsigned CCMask = MI->getOperand(4).getImm(); 2291 DebugLoc DL = MI->getDebugLoc(); 2292 2293 MachineBasicBlock *StartMBB = MBB; 2294 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2295 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2296 2297 // StartMBB: 2298 // BRC CCMask, JoinMBB 2299 // # fallthrough to FalseMBB 2300 MBB = StartMBB; 2301 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2302 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2303 MBB->addSuccessor(JoinMBB); 2304 MBB->addSuccessor(FalseMBB); 2305 2306 // FalseMBB: 2307 // # fallthrough to JoinMBB 2308 MBB = FalseMBB; 2309 MBB->addSuccessor(JoinMBB); 2310 2311 // JoinMBB: 2312 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 2313 // ... 2314 MBB = JoinMBB; 2315 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg) 2316 .addReg(TrueReg).addMBB(StartMBB) 2317 .addReg(FalseReg).addMBB(FalseMBB); 2318 2319 MI->eraseFromParent(); 2320 return JoinMBB; 2321} 2322 2323// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 2324// StoreOpcode is the store to use and Invert says whether the store should 2325// happen when the condition is false rather than true. If a STORE ON 2326// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 2327MachineBasicBlock * 2328SystemZTargetLowering::emitCondStore(MachineInstr *MI, 2329 MachineBasicBlock *MBB, 2330 unsigned StoreOpcode, unsigned STOCOpcode, 2331 bool Invert) const { 2332 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2333 2334 unsigned SrcReg = MI->getOperand(0).getReg(); 2335 MachineOperand Base = MI->getOperand(1); 2336 int64_t Disp = MI->getOperand(2).getImm(); 2337 unsigned IndexReg = MI->getOperand(3).getReg(); 2338 unsigned CCValid = MI->getOperand(4).getImm(); 2339 unsigned CCMask = MI->getOperand(5).getImm(); 2340 DebugLoc DL = MI->getDebugLoc(); 2341 2342 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 2343 2344 // Use STOCOpcode if possible. We could use different store patterns in 2345 // order to avoid matching the index register, but the performance trade-offs 2346 // might be more complicated in that case. 2347 if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) { 2348 if (Invert) 2349 CCMask ^= CCValid; 2350 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 2351 .addReg(SrcReg).addOperand(Base).addImm(Disp) 2352 .addImm(CCValid).addImm(CCMask); 2353 MI->eraseFromParent(); 2354 return MBB; 2355 } 2356 2357 // Get the condition needed to branch around the store. 2358 if (!Invert) 2359 CCMask ^= CCValid; 2360 2361 MachineBasicBlock *StartMBB = MBB; 2362 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2363 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2364 2365 // StartMBB: 2366 // BRC CCMask, JoinMBB 2367 // # fallthrough to FalseMBB 2368 MBB = StartMBB; 2369 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2370 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2371 MBB->addSuccessor(JoinMBB); 2372 MBB->addSuccessor(FalseMBB); 2373 2374 // FalseMBB: 2375 // store %SrcReg, %Disp(%Index,%Base) 2376 // # fallthrough to JoinMBB 2377 MBB = FalseMBB; 2378 BuildMI(MBB, DL, TII->get(StoreOpcode)) 2379 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 2380 MBB->addSuccessor(JoinMBB); 2381 2382 MI->eraseFromParent(); 2383 return JoinMBB; 2384} 2385 2386// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 2387// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 2388// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 2389// BitSize is the width of the field in bits, or 0 if this is a partword 2390// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 2391// is one of the operands. Invert says whether the field should be 2392// inverted after performing BinOpcode (e.g. for NAND). 2393MachineBasicBlock * 2394SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 2395 MachineBasicBlock *MBB, 2396 unsigned BinOpcode, 2397 unsigned BitSize, 2398 bool Invert) const { 2399 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2400 MachineFunction &MF = *MBB->getParent(); 2401 MachineRegisterInfo &MRI = MF.getRegInfo(); 2402 bool IsSubWord = (BitSize < 32); 2403 2404 // Extract the operands. Base can be a register or a frame index. 2405 // Src2 can be a register or immediate. 2406 unsigned Dest = MI->getOperand(0).getReg(); 2407 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2408 int64_t Disp = MI->getOperand(2).getImm(); 2409 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2410 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2411 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2412 DebugLoc DL = MI->getDebugLoc(); 2413 if (IsSubWord) 2414 BitSize = MI->getOperand(6).getImm(); 2415 2416 // Subword operations use 32-bit registers. 2417 const TargetRegisterClass *RC = (BitSize <= 32 ? 2418 &SystemZ::GR32BitRegClass : 2419 &SystemZ::GR64BitRegClass); 2420 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2421 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2422 2423 // Get the right opcodes for the displacement. 2424 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2425 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2426 assert(LOpcode && CSOpcode && "Displacement out of range"); 2427 2428 // Create virtual registers for temporary results. 2429 unsigned OrigVal = MRI.createVirtualRegister(RC); 2430 unsigned OldVal = MRI.createVirtualRegister(RC); 2431 unsigned NewVal = (BinOpcode || IsSubWord ? 2432 MRI.createVirtualRegister(RC) : Src2.getReg()); 2433 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2434 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2435 2436 // Insert a basic block for the main loop. 2437 MachineBasicBlock *StartMBB = MBB; 2438 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2439 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2440 2441 // StartMBB: 2442 // ... 2443 // %OrigVal = L Disp(%Base) 2444 // # fall through to LoopMMB 2445 MBB = StartMBB; 2446 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2447 .addOperand(Base).addImm(Disp).addReg(0); 2448 MBB->addSuccessor(LoopMBB); 2449 2450 // LoopMBB: 2451 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2452 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2453 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2454 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2455 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2456 // JNE LoopMBB 2457 // # fall through to DoneMMB 2458 MBB = LoopMBB; 2459 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2460 .addReg(OrigVal).addMBB(StartMBB) 2461 .addReg(Dest).addMBB(LoopMBB); 2462 if (IsSubWord) 2463 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2464 .addReg(OldVal).addReg(BitShift).addImm(0); 2465 if (Invert) { 2466 // Perform the operation normally and then invert every bit of the field. 2467 unsigned Tmp = MRI.createVirtualRegister(RC); 2468 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2469 .addReg(RotatedOldVal).addOperand(Src2); 2470 if (BitSize < 32) 2471 // XILF with the upper BitSize bits set. 2472 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2473 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 2474 else if (BitSize == 32) 2475 // XILF with every bit set. 2476 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2477 .addReg(Tmp).addImm(~uint32_t(0)); 2478 else { 2479 // Use LCGR and add -1 to the result, which is more compact than 2480 // an XILF, XILH pair. 2481 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2482 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2483 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2484 .addReg(Tmp2).addImm(-1); 2485 } 2486 } else if (BinOpcode) 2487 // A simply binary operation. 2488 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2489 .addReg(RotatedOldVal).addOperand(Src2); 2490 else if (IsSubWord) 2491 // Use RISBG to rotate Src2 into position and use it to replace the 2492 // field in RotatedOldVal. 2493 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2494 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2495 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2496 if (IsSubWord) 2497 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2498 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2499 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2500 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2501 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2502 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2503 MBB->addSuccessor(LoopMBB); 2504 MBB->addSuccessor(DoneMBB); 2505 2506 MI->eraseFromParent(); 2507 return DoneMBB; 2508} 2509 2510// Implement EmitInstrWithCustomInserter for pseudo 2511// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2512// instruction that should be used to compare the current field with the 2513// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2514// for when the current field should be kept. BitSize is the width of 2515// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2516MachineBasicBlock * 2517SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2518 MachineBasicBlock *MBB, 2519 unsigned CompareOpcode, 2520 unsigned KeepOldMask, 2521 unsigned BitSize) const { 2522 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2523 MachineFunction &MF = *MBB->getParent(); 2524 MachineRegisterInfo &MRI = MF.getRegInfo(); 2525 bool IsSubWord = (BitSize < 32); 2526 2527 // Extract the operands. Base can be a register or a frame index. 2528 unsigned Dest = MI->getOperand(0).getReg(); 2529 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2530 int64_t Disp = MI->getOperand(2).getImm(); 2531 unsigned Src2 = MI->getOperand(3).getReg(); 2532 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2533 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2534 DebugLoc DL = MI->getDebugLoc(); 2535 if (IsSubWord) 2536 BitSize = MI->getOperand(6).getImm(); 2537 2538 // Subword operations use 32-bit registers. 2539 const TargetRegisterClass *RC = (BitSize <= 32 ? 2540 &SystemZ::GR32BitRegClass : 2541 &SystemZ::GR64BitRegClass); 2542 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2543 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2544 2545 // Get the right opcodes for the displacement. 2546 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2547 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2548 assert(LOpcode && CSOpcode && "Displacement out of range"); 2549 2550 // Create virtual registers for temporary results. 2551 unsigned OrigVal = MRI.createVirtualRegister(RC); 2552 unsigned OldVal = MRI.createVirtualRegister(RC); 2553 unsigned NewVal = MRI.createVirtualRegister(RC); 2554 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2555 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2556 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2557 2558 // Insert 3 basic blocks for the loop. 2559 MachineBasicBlock *StartMBB = MBB; 2560 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2561 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2562 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2563 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2564 2565 // StartMBB: 2566 // ... 2567 // %OrigVal = L Disp(%Base) 2568 // # fall through to LoopMMB 2569 MBB = StartMBB; 2570 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2571 .addOperand(Base).addImm(Disp).addReg(0); 2572 MBB->addSuccessor(LoopMBB); 2573 2574 // LoopMBB: 2575 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2576 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2577 // CompareOpcode %RotatedOldVal, %Src2 2578 // BRC KeepOldMask, UpdateMBB 2579 MBB = LoopMBB; 2580 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2581 .addReg(OrigVal).addMBB(StartMBB) 2582 .addReg(Dest).addMBB(UpdateMBB); 2583 if (IsSubWord) 2584 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2585 .addReg(OldVal).addReg(BitShift).addImm(0); 2586 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2587 .addReg(RotatedOldVal).addReg(Src2); 2588 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2589 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2590 MBB->addSuccessor(UpdateMBB); 2591 MBB->addSuccessor(UseAltMBB); 2592 2593 // UseAltMBB: 2594 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2595 // # fall through to UpdateMMB 2596 MBB = UseAltMBB; 2597 if (IsSubWord) 2598 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2599 .addReg(RotatedOldVal).addReg(Src2) 2600 .addImm(32).addImm(31 + BitSize).addImm(0); 2601 MBB->addSuccessor(UpdateMBB); 2602 2603 // UpdateMBB: 2604 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2605 // [ %RotatedAltVal, UseAltMBB ] 2606 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2607 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2608 // JNE LoopMBB 2609 // # fall through to DoneMMB 2610 MBB = UpdateMBB; 2611 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2612 .addReg(RotatedOldVal).addMBB(LoopMBB) 2613 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2614 if (IsSubWord) 2615 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2616 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2617 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2618 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2619 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2620 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2621 MBB->addSuccessor(LoopMBB); 2622 MBB->addSuccessor(DoneMBB); 2623 2624 MI->eraseFromParent(); 2625 return DoneMBB; 2626} 2627 2628// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2629// instruction MI. 2630MachineBasicBlock * 2631SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2632 MachineBasicBlock *MBB) const { 2633 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2634 MachineFunction &MF = *MBB->getParent(); 2635 MachineRegisterInfo &MRI = MF.getRegInfo(); 2636 2637 // Extract the operands. Base can be a register or a frame index. 2638 unsigned Dest = MI->getOperand(0).getReg(); 2639 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2640 int64_t Disp = MI->getOperand(2).getImm(); 2641 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2642 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2643 unsigned BitShift = MI->getOperand(5).getReg(); 2644 unsigned NegBitShift = MI->getOperand(6).getReg(); 2645 int64_t BitSize = MI->getOperand(7).getImm(); 2646 DebugLoc DL = MI->getDebugLoc(); 2647 2648 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2649 2650 // Get the right opcodes for the displacement. 2651 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2652 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2653 assert(LOpcode && CSOpcode && "Displacement out of range"); 2654 2655 // Create virtual registers for temporary results. 2656 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2657 unsigned OldVal = MRI.createVirtualRegister(RC); 2658 unsigned CmpVal = MRI.createVirtualRegister(RC); 2659 unsigned SwapVal = MRI.createVirtualRegister(RC); 2660 unsigned StoreVal = MRI.createVirtualRegister(RC); 2661 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2662 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2663 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2664 2665 // Insert 2 basic blocks for the loop. 2666 MachineBasicBlock *StartMBB = MBB; 2667 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2668 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2669 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 2670 2671 // StartMBB: 2672 // ... 2673 // %OrigOldVal = L Disp(%Base) 2674 // # fall through to LoopMMB 2675 MBB = StartMBB; 2676 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 2677 .addOperand(Base).addImm(Disp).addReg(0); 2678 MBB->addSuccessor(LoopMBB); 2679 2680 // LoopMBB: 2681 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 2682 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 2683 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 2684 // %Dest = RLL %OldVal, BitSize(%BitShift) 2685 // ^^ The low BitSize bits contain the field 2686 // of interest. 2687 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 2688 // ^^ Replace the upper 32-BitSize bits of the 2689 // comparison value with those that we loaded, 2690 // so that we can use a full word comparison. 2691 // CR %Dest, %RetryCmpVal 2692 // JNE DoneMBB 2693 // # Fall through to SetMBB 2694 MBB = LoopMBB; 2695 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2696 .addReg(OrigOldVal).addMBB(StartMBB) 2697 .addReg(RetryOldVal).addMBB(SetMBB); 2698 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 2699 .addReg(OrigCmpVal).addMBB(StartMBB) 2700 .addReg(RetryCmpVal).addMBB(SetMBB); 2701 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 2702 .addReg(OrigSwapVal).addMBB(StartMBB) 2703 .addReg(RetrySwapVal).addMBB(SetMBB); 2704 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 2705 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 2706 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 2707 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2708 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 2709 .addReg(Dest).addReg(RetryCmpVal); 2710 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2711 .addImm(SystemZ::CCMASK_ICMP) 2712 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 2713 MBB->addSuccessor(DoneMBB); 2714 MBB->addSuccessor(SetMBB); 2715 2716 // SetMBB: 2717 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 2718 // ^^ Replace the upper 32-BitSize bits of the new 2719 // value with those that we loaded. 2720 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 2721 // ^^ Rotate the new field to its proper position. 2722 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 2723 // JNE LoopMBB 2724 // # fall through to ExitMMB 2725 MBB = SetMBB; 2726 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 2727 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 2728 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 2729 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 2730 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2731 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2732 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2733 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2734 MBB->addSuccessor(LoopMBB); 2735 MBB->addSuccessor(DoneMBB); 2736 2737 MI->eraseFromParent(); 2738 return DoneMBB; 2739} 2740 2741// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2742// if the high register of the GR128 value must be cleared or false if 2743// it's "don't care". SubReg is subreg_l32 when extending a GR32 2744// and subreg_l64 when extending a GR64. 2745MachineBasicBlock * 2746SystemZTargetLowering::emitExt128(MachineInstr *MI, 2747 MachineBasicBlock *MBB, 2748 bool ClearEven, unsigned SubReg) const { 2749 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2750 MachineFunction &MF = *MBB->getParent(); 2751 MachineRegisterInfo &MRI = MF.getRegInfo(); 2752 DebugLoc DL = MI->getDebugLoc(); 2753 2754 unsigned Dest = MI->getOperand(0).getReg(); 2755 unsigned Src = MI->getOperand(1).getReg(); 2756 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2757 2758 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2759 if (ClearEven) { 2760 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2761 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2762 2763 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2764 .addImm(0); 2765 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2766 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64); 2767 In128 = NewIn128; 2768 } 2769 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2770 .addReg(In128).addReg(Src).addImm(SubReg); 2771 2772 MI->eraseFromParent(); 2773 return MBB; 2774} 2775 2776MachineBasicBlock * 2777SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 2778 MachineBasicBlock *MBB, 2779 unsigned Opcode) const { 2780 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2781 MachineFunction &MF = *MBB->getParent(); 2782 MachineRegisterInfo &MRI = MF.getRegInfo(); 2783 DebugLoc DL = MI->getDebugLoc(); 2784 2785 MachineOperand DestBase = earlyUseOperand(MI->getOperand(0)); 2786 uint64_t DestDisp = MI->getOperand(1).getImm(); 2787 MachineOperand SrcBase = earlyUseOperand(MI->getOperand(2)); 2788 uint64_t SrcDisp = MI->getOperand(3).getImm(); 2789 uint64_t Length = MI->getOperand(4).getImm(); 2790 2791 // When generating more than one CLC, all but the last will need to 2792 // branch to the end when a difference is found. 2793 MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ? 2794 splitBlockAfter(MI, MBB) : 0); 2795 2796 // Check for the loop form, in which operand 5 is the trip count. 2797 if (MI->getNumExplicitOperands() > 5) { 2798 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); 2799 2800 uint64_t StartCountReg = MI->getOperand(5).getReg(); 2801 uint64_t StartSrcReg = forceReg(MI, SrcBase, TII); 2802 uint64_t StartDestReg = (HaveSingleBase ? StartSrcReg : 2803 forceReg(MI, DestBase, TII)); 2804 2805 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass; 2806 uint64_t ThisSrcReg = MRI.createVirtualRegister(RC); 2807 uint64_t ThisDestReg = (HaveSingleBase ? ThisSrcReg : 2808 MRI.createVirtualRegister(RC)); 2809 uint64_t NextSrcReg = MRI.createVirtualRegister(RC); 2810 uint64_t NextDestReg = (HaveSingleBase ? NextSrcReg : 2811 MRI.createVirtualRegister(RC)); 2812 2813 RC = &SystemZ::GR64BitRegClass; 2814 uint64_t ThisCountReg = MRI.createVirtualRegister(RC); 2815 uint64_t NextCountReg = MRI.createVirtualRegister(RC); 2816 2817 MachineBasicBlock *StartMBB = MBB; 2818 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2819 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2820 MachineBasicBlock *NextMBB = (EndMBB ? emitBlockAfter(LoopMBB) : LoopMBB); 2821 2822 // StartMBB: 2823 // # fall through to LoopMMB 2824 MBB->addSuccessor(LoopMBB); 2825 2826 // LoopMBB: 2827 // %ThisDestReg = phi [ %StartDestReg, StartMBB ], 2828 // [ %NextDestReg, NextMBB ] 2829 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ], 2830 // [ %NextSrcReg, NextMBB ] 2831 // %ThisCountReg = phi [ %StartCountReg, StartMBB ], 2832 // [ %NextCountReg, NextMBB ] 2833 // ( PFD 2, 768+DestDisp(%ThisDestReg) ) 2834 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg) 2835 // ( JLH EndMBB ) 2836 // 2837 // The prefetch is used only for MVC. The JLH is used only for CLC. 2838 MBB = LoopMBB; 2839 2840 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) 2841 .addReg(StartDestReg).addMBB(StartMBB) 2842 .addReg(NextDestReg).addMBB(NextMBB); 2843 if (!HaveSingleBase) 2844 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) 2845 .addReg(StartSrcReg).addMBB(StartMBB) 2846 .addReg(NextSrcReg).addMBB(NextMBB); 2847 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) 2848 .addReg(StartCountReg).addMBB(StartMBB) 2849 .addReg(NextCountReg).addMBB(NextMBB); 2850 if (Opcode == SystemZ::MVC) 2851 BuildMI(MBB, DL, TII->get(SystemZ::PFD)) 2852 .addImm(SystemZ::PFD_WRITE) 2853 .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); 2854 BuildMI(MBB, DL, TII->get(Opcode)) 2855 .addReg(ThisDestReg).addImm(DestDisp).addImm(256) 2856 .addReg(ThisSrcReg).addImm(SrcDisp); 2857 if (EndMBB) { 2858 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2859 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2860 .addMBB(EndMBB); 2861 MBB->addSuccessor(EndMBB); 2862 MBB->addSuccessor(NextMBB); 2863 } 2864 2865 // NextMBB: 2866 // %NextDestReg = LA 256(%ThisDestReg) 2867 // %NextSrcReg = LA 256(%ThisSrcReg) 2868 // %NextCountReg = AGHI %ThisCountReg, -1 2869 // CGHI %NextCountReg, 0 2870 // JLH LoopMBB 2871 // # fall through to DoneMMB 2872 // 2873 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes. 2874 MBB = NextMBB; 2875 2876 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) 2877 .addReg(ThisDestReg).addImm(256).addReg(0); 2878 if (!HaveSingleBase) 2879 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) 2880 .addReg(ThisSrcReg).addImm(256).addReg(0); 2881 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) 2882 .addReg(ThisCountReg).addImm(-1); 2883 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) 2884 .addReg(NextCountReg).addImm(0); 2885 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2886 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2887 .addMBB(LoopMBB); 2888 MBB->addSuccessor(LoopMBB); 2889 MBB->addSuccessor(DoneMBB); 2890 2891 DestBase = MachineOperand::CreateReg(NextDestReg, false); 2892 SrcBase = MachineOperand::CreateReg(NextSrcReg, false); 2893 Length &= 255; 2894 MBB = DoneMBB; 2895 } 2896 // Handle any remaining bytes with straight-line code. 2897 while (Length > 0) { 2898 uint64_t ThisLength = std::min(Length, uint64_t(256)); 2899 // The previous iteration might have created out-of-range displacements. 2900 // Apply them using LAY if so. 2901 if (!isUInt<12>(DestDisp)) { 2902 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2903 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 2904 .addOperand(DestBase).addImm(DestDisp).addReg(0); 2905 DestBase = MachineOperand::CreateReg(Reg, false); 2906 DestDisp = 0; 2907 } 2908 if (!isUInt<12>(SrcDisp)) { 2909 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2910 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 2911 .addOperand(SrcBase).addImm(SrcDisp).addReg(0); 2912 SrcBase = MachineOperand::CreateReg(Reg, false); 2913 SrcDisp = 0; 2914 } 2915 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 2916 .addOperand(DestBase).addImm(DestDisp).addImm(ThisLength) 2917 .addOperand(SrcBase).addImm(SrcDisp); 2918 DestDisp += ThisLength; 2919 SrcDisp += ThisLength; 2920 Length -= ThisLength; 2921 // If there's another CLC to go, branch to the end if a difference 2922 // was found. 2923 if (EndMBB && Length > 0) { 2924 MachineBasicBlock *NextMBB = splitBlockBefore(MI, MBB); 2925 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2926 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 2927 .addMBB(EndMBB); 2928 MBB->addSuccessor(EndMBB); 2929 MBB->addSuccessor(NextMBB); 2930 MBB = NextMBB; 2931 } 2932 } 2933 if (EndMBB) { 2934 MBB->addSuccessor(EndMBB); 2935 MBB = EndMBB; 2936 MBB->addLiveIn(SystemZ::CC); 2937 } 2938 2939 MI->eraseFromParent(); 2940 return MBB; 2941} 2942 2943// Decompose string pseudo-instruction MI into a loop that continually performs 2944// Opcode until CC != 3. 2945MachineBasicBlock * 2946SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 2947 MachineBasicBlock *MBB, 2948 unsigned Opcode) const { 2949 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2950 MachineFunction &MF = *MBB->getParent(); 2951 MachineRegisterInfo &MRI = MF.getRegInfo(); 2952 DebugLoc DL = MI->getDebugLoc(); 2953 2954 uint64_t End1Reg = MI->getOperand(0).getReg(); 2955 uint64_t Start1Reg = MI->getOperand(1).getReg(); 2956 uint64_t Start2Reg = MI->getOperand(2).getReg(); 2957 uint64_t CharReg = MI->getOperand(3).getReg(); 2958 2959 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 2960 uint64_t This1Reg = MRI.createVirtualRegister(RC); 2961 uint64_t This2Reg = MRI.createVirtualRegister(RC); 2962 uint64_t End2Reg = MRI.createVirtualRegister(RC); 2963 2964 MachineBasicBlock *StartMBB = MBB; 2965 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2966 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2967 2968 // StartMBB: 2969 // # fall through to LoopMMB 2970 MBB->addSuccessor(LoopMBB); 2971 2972 // LoopMBB: 2973 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 2974 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 2975 // R0L = %CharReg 2976 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L 2977 // JO LoopMBB 2978 // # fall through to DoneMMB 2979 // 2980 // The load of R0L can be hoisted by post-RA LICM. 2981 MBB = LoopMBB; 2982 2983 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 2984 .addReg(Start1Reg).addMBB(StartMBB) 2985 .addReg(End1Reg).addMBB(LoopMBB); 2986 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 2987 .addReg(Start2Reg).addMBB(StartMBB) 2988 .addReg(End2Reg).addMBB(LoopMBB); 2989 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); 2990 BuildMI(MBB, DL, TII->get(Opcode)) 2991 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 2992 .addReg(This1Reg).addReg(This2Reg); 2993 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2994 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 2995 MBB->addSuccessor(LoopMBB); 2996 MBB->addSuccessor(DoneMBB); 2997 2998 DoneMBB->addLiveIn(SystemZ::CC); 2999 3000 MI->eraseFromParent(); 3001 return DoneMBB; 3002} 3003 3004MachineBasicBlock *SystemZTargetLowering:: 3005EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 3006 switch (MI->getOpcode()) { 3007 case SystemZ::Select32Mux: 3008 case SystemZ::Select32: 3009 case SystemZ::SelectF32: 3010 case SystemZ::Select64: 3011 case SystemZ::SelectF64: 3012 case SystemZ::SelectF128: 3013 return emitSelect(MI, MBB); 3014 3015 case SystemZ::CondStore8Mux: 3016 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false); 3017 case SystemZ::CondStore8MuxInv: 3018 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true); 3019 case SystemZ::CondStore16Mux: 3020 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false); 3021 case SystemZ::CondStore16MuxInv: 3022 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true); 3023 case SystemZ::CondStore8: 3024 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 3025 case SystemZ::CondStore8Inv: 3026 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 3027 case SystemZ::CondStore16: 3028 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 3029 case SystemZ::CondStore16Inv: 3030 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 3031 case SystemZ::CondStore32: 3032 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 3033 case SystemZ::CondStore32Inv: 3034 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 3035 case SystemZ::CondStore64: 3036 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 3037 case SystemZ::CondStore64Inv: 3038 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 3039 case SystemZ::CondStoreF32: 3040 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 3041 case SystemZ::CondStoreF32Inv: 3042 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 3043 case SystemZ::CondStoreF64: 3044 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 3045 case SystemZ::CondStoreF64Inv: 3046 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 3047 3048 case SystemZ::AEXT128_64: 3049 return emitExt128(MI, MBB, false, SystemZ::subreg_l64); 3050 case SystemZ::ZEXT128_32: 3051 return emitExt128(MI, MBB, true, SystemZ::subreg_l32); 3052 case SystemZ::ZEXT128_64: 3053 return emitExt128(MI, MBB, true, SystemZ::subreg_l64); 3054 3055 case SystemZ::ATOMIC_SWAPW: 3056 return emitAtomicLoadBinary(MI, MBB, 0, 0); 3057 case SystemZ::ATOMIC_SWAP_32: 3058 return emitAtomicLoadBinary(MI, MBB, 0, 32); 3059 case SystemZ::ATOMIC_SWAP_64: 3060 return emitAtomicLoadBinary(MI, MBB, 0, 64); 3061 3062 case SystemZ::ATOMIC_LOADW_AR: 3063 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 3064 case SystemZ::ATOMIC_LOADW_AFI: 3065 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 3066 case SystemZ::ATOMIC_LOAD_AR: 3067 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 3068 case SystemZ::ATOMIC_LOAD_AHI: 3069 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 3070 case SystemZ::ATOMIC_LOAD_AFI: 3071 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 3072 case SystemZ::ATOMIC_LOAD_AGR: 3073 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 3074 case SystemZ::ATOMIC_LOAD_AGHI: 3075 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 3076 case SystemZ::ATOMIC_LOAD_AGFI: 3077 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 3078 3079 case SystemZ::ATOMIC_LOADW_SR: 3080 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 3081 case SystemZ::ATOMIC_LOAD_SR: 3082 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 3083 case SystemZ::ATOMIC_LOAD_SGR: 3084 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 3085 3086 case SystemZ::ATOMIC_LOADW_NR: 3087 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 3088 case SystemZ::ATOMIC_LOADW_NILH: 3089 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); 3090 case SystemZ::ATOMIC_LOAD_NR: 3091 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 3092 case SystemZ::ATOMIC_LOAD_NILL: 3093 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); 3094 case SystemZ::ATOMIC_LOAD_NILH: 3095 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); 3096 case SystemZ::ATOMIC_LOAD_NILF: 3097 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); 3098 case SystemZ::ATOMIC_LOAD_NGR: 3099 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 3100 case SystemZ::ATOMIC_LOAD_NILL64: 3101 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); 3102 case SystemZ::ATOMIC_LOAD_NILH64: 3103 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); 3104 case SystemZ::ATOMIC_LOAD_NIHL64: 3105 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); 3106 case SystemZ::ATOMIC_LOAD_NIHH64: 3107 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); 3108 case SystemZ::ATOMIC_LOAD_NILF64: 3109 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); 3110 case SystemZ::ATOMIC_LOAD_NIHF64: 3111 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); 3112 3113 case SystemZ::ATOMIC_LOADW_OR: 3114 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 3115 case SystemZ::ATOMIC_LOADW_OILH: 3116 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); 3117 case SystemZ::ATOMIC_LOAD_OR: 3118 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 3119 case SystemZ::ATOMIC_LOAD_OILL: 3120 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); 3121 case SystemZ::ATOMIC_LOAD_OILH: 3122 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); 3123 case SystemZ::ATOMIC_LOAD_OILF: 3124 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); 3125 case SystemZ::ATOMIC_LOAD_OGR: 3126 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 3127 case SystemZ::ATOMIC_LOAD_OILL64: 3128 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); 3129 case SystemZ::ATOMIC_LOAD_OILH64: 3130 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); 3131 case SystemZ::ATOMIC_LOAD_OIHL64: 3132 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); 3133 case SystemZ::ATOMIC_LOAD_OIHH64: 3134 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); 3135 case SystemZ::ATOMIC_LOAD_OILF64: 3136 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); 3137 case SystemZ::ATOMIC_LOAD_OIHF64: 3138 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); 3139 3140 case SystemZ::ATOMIC_LOADW_XR: 3141 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 3142 case SystemZ::ATOMIC_LOADW_XILF: 3143 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); 3144 case SystemZ::ATOMIC_LOAD_XR: 3145 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 3146 case SystemZ::ATOMIC_LOAD_XILF: 3147 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); 3148 case SystemZ::ATOMIC_LOAD_XGR: 3149 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 3150 case SystemZ::ATOMIC_LOAD_XILF64: 3151 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); 3152 case SystemZ::ATOMIC_LOAD_XIHF64: 3153 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); 3154 3155 case SystemZ::ATOMIC_LOADW_NRi: 3156 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 3157 case SystemZ::ATOMIC_LOADW_NILHi: 3158 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); 3159 case SystemZ::ATOMIC_LOAD_NRi: 3160 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 3161 case SystemZ::ATOMIC_LOAD_NILLi: 3162 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); 3163 case SystemZ::ATOMIC_LOAD_NILHi: 3164 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); 3165 case SystemZ::ATOMIC_LOAD_NILFi: 3166 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); 3167 case SystemZ::ATOMIC_LOAD_NGRi: 3168 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 3169 case SystemZ::ATOMIC_LOAD_NILL64i: 3170 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); 3171 case SystemZ::ATOMIC_LOAD_NILH64i: 3172 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); 3173 case SystemZ::ATOMIC_LOAD_NIHL64i: 3174 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); 3175 case SystemZ::ATOMIC_LOAD_NIHH64i: 3176 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); 3177 case SystemZ::ATOMIC_LOAD_NILF64i: 3178 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); 3179 case SystemZ::ATOMIC_LOAD_NIHF64i: 3180 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); 3181 3182 case SystemZ::ATOMIC_LOADW_MIN: 3183 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3184 SystemZ::CCMASK_CMP_LE, 0); 3185 case SystemZ::ATOMIC_LOAD_MIN_32: 3186 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3187 SystemZ::CCMASK_CMP_LE, 32); 3188 case SystemZ::ATOMIC_LOAD_MIN_64: 3189 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3190 SystemZ::CCMASK_CMP_LE, 64); 3191 3192 case SystemZ::ATOMIC_LOADW_MAX: 3193 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3194 SystemZ::CCMASK_CMP_GE, 0); 3195 case SystemZ::ATOMIC_LOAD_MAX_32: 3196 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3197 SystemZ::CCMASK_CMP_GE, 32); 3198 case SystemZ::ATOMIC_LOAD_MAX_64: 3199 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3200 SystemZ::CCMASK_CMP_GE, 64); 3201 3202 case SystemZ::ATOMIC_LOADW_UMIN: 3203 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3204 SystemZ::CCMASK_CMP_LE, 0); 3205 case SystemZ::ATOMIC_LOAD_UMIN_32: 3206 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3207 SystemZ::CCMASK_CMP_LE, 32); 3208 case SystemZ::ATOMIC_LOAD_UMIN_64: 3209 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3210 SystemZ::CCMASK_CMP_LE, 64); 3211 3212 case SystemZ::ATOMIC_LOADW_UMAX: 3213 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3214 SystemZ::CCMASK_CMP_GE, 0); 3215 case SystemZ::ATOMIC_LOAD_UMAX_32: 3216 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3217 SystemZ::CCMASK_CMP_GE, 32); 3218 case SystemZ::ATOMIC_LOAD_UMAX_64: 3219 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3220 SystemZ::CCMASK_CMP_GE, 64); 3221 3222 case SystemZ::ATOMIC_CMP_SWAPW: 3223 return emitAtomicCmpSwapW(MI, MBB); 3224 case SystemZ::MVCSequence: 3225 case SystemZ::MVCLoop: 3226 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 3227 case SystemZ::NCSequence: 3228 case SystemZ::NCLoop: 3229 return emitMemMemWrapper(MI, MBB, SystemZ::NC); 3230 case SystemZ::OCSequence: 3231 case SystemZ::OCLoop: 3232 return emitMemMemWrapper(MI, MBB, SystemZ::OC); 3233 case SystemZ::XCSequence: 3234 case SystemZ::XCLoop: 3235 return emitMemMemWrapper(MI, MBB, SystemZ::XC); 3236 case SystemZ::CLCSequence: 3237 case SystemZ::CLCLoop: 3238 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 3239 case SystemZ::CLSTLoop: 3240 return emitStringWrapper(MI, MBB, SystemZ::CLST); 3241 case SystemZ::MVSTLoop: 3242 return emitStringWrapper(MI, MBB, SystemZ::MVST); 3243 case SystemZ::SRSTLoop: 3244 return emitStringWrapper(MI, MBB, SystemZ::SRST); 3245 default: 3246 llvm_unreachable("Unexpected instr type to insert"); 3247 } 3248} 3249