11d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 21d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// 31d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// The LLVM Compiler Infrastructure 41d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// 51d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// This file is distributed under the University of Illinois Open Source 61d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// License. See LICENSE.TXT for details. 71d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// 81d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 91d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// 101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// This file implements the SystemZTargetLowering class. 111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// 121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZISelLowering.h" 151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZCallingConv.h" 161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZConstantPoolValue.h" 171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZMachineFunctionInfo.h" 181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZTargetMachine.h" 191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "llvm/CodeGen/CallingConvLower.h" 201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "llvm/CodeGen/MachineInstrBuilder.h" 211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "llvm/CodeGen/MachineRegisterInfo.h" 221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar#include "llvm/IR/Intrinsics.h" 24e3ba15c794839abe076e3e2bdf6c626396a19d4dWill Dietz#include <cctype> 25e3ba15c794839abe076e3e2bdf6c626396a19d4dWill Dietz 261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandusing namespace llvm; 271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "systemz-lower" 29dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 30d1a4f579bf45aec933c79292b6b9663581438738Richard Sandifordnamespace { 31d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford// Represents a sequence for extracting a 0/1 value from an IPM result: 32d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford// (((X ^ XORValue) + AddValue) >> Bit) 33d1a4f579bf45aec933c79292b6b9663581438738Richard Sandifordstruct IPMConversion { 34d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 35d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 36d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 37d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford int64_t XORValue; 38d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford int64_t AddValue; 39d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford unsigned Bit; 40d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford}; 4136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Represents information about a comparison. 4336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct Comparison { 4436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Comparison(SDValue Op0In, SDValue Op1In) 4536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {} 4636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The operands to the comparison. 4836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Op0, Op1; 4936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The opcode that should be used to compare Op0 and Op1. 5136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Opcode; 5236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // A SystemZICMP value. Only used for integer comparisons. 5436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ICmpType; 5536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The mask of CC values that Opcode can produce. 5736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CCValid; 5836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The mask of CC values for which the original condition is true. 6036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CCMask; 6136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} // end anonymous namespace 63d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Classify VT as either 32 or 64 bit. 651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandstatic bool is32Bit(EVT VT) { 661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (VT.getSimpleVT().SimpleTy) { 671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::i32: 681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return true; 691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::i64: 701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return false; 711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unsupported type"); 731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Return a version of MachineOperand that can be safely used before the 771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// final use. 781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandstatic MachineOperand earlyUseOperand(MachineOperand Op) { 791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Op.isReg()) 801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.setIsKill(false); 811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 84f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarSystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, 85ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SystemZSubtarget &STI) 86f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar : TargetLowering(TM), Subtarget(STI) { 87f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize()); 881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Set up the register classes. 9055d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford if (Subtarget.hasHighWord()) 9155d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 9255d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford else 9355d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass); 976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass); 986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 1006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 1016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 1021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 1031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 1056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass); 1066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass); 1076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass); 1086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass); 1096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass); 1106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass); 1116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 1126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Compute derived properties from the register classes 114ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines computeRegisterProperties(Subtarget.getRegisterInfo()); 1151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Set up special registers. 1171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setStackPointerRegisterToSaveRestore(SystemZ::R15D); 1181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // TODO: It may be better to default to latency-oriented scheduling, however 1201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // LLVM's current latency-oriented scheduler can't handle physreg definitions 121dbd8eb26ce1e7de9b69f5c46f45ba011a706c9b9Richard Sandiford // such as SystemZ has with CC, so set this to the register-pressure 1221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // scheduler, because it can. 1231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setSchedulingPreference(Sched::RegPressure); 1241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setBooleanContents(ZeroOrOneBooleanContent); 1266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 1271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Instructions are strings of 2-byte aligned 2-byte values. 1291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setMinFunctionAlignment(2); 1301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Handle operations that are handled in a similar way for all types. 1321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 1331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand I <= MVT::LAST_FP_VALUETYPE; 1341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ++I) { 1351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MVT VT = MVT::SimpleValueType(I); 1361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isTypeLegal(VT)) { 137d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // Lower SET_CC into an IPM-based sequence. 138d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford setOperationAction(ISD::SETCC, VT, Custom); 1391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 1411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SELECT, VT, Expand); 1421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Lower SELECT_CC and BR_CC into separate comparisons and branches. 1441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SELECT_CC, VT, Custom); 1451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::BR_CC, VT, Custom); 1461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 1471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 1481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Expand jump table branches as address arithmetic followed by an 1501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // indirect jump. 1511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::BR_JT, MVT::Other, Expand); 1521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Expand BRCOND into a BR_CC (see above). 1541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::BRCOND, MVT::Other, Expand); 1551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Handle integer types. 1571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 1581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand I <= MVT::LAST_INTEGER_VALUETYPE; 1591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ++I) { 1601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MVT VT = MVT::SimpleValueType(I); 1611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isTypeLegal(VT)) { 1621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Expand individual DIV and REMs into DIVREMs. 1631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SDIV, VT, Expand); 1641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::UDIV, VT, Expand); 1651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SREM, VT, Expand); 1661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::UREM, VT, Expand); 1671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SDIVREM, VT, Custom); 1681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::UDIVREM, VT, Custom); 1691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 17036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and 17136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // stores, putting a serialization instruction after the stores. 17236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::ATOMIC_LOAD, VT, Custom); 17336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 17436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 17536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are 17636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // available, or if the operand is constant. 17736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 1781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1790c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Use POPCNT on z196 and above. 1800c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Subtarget.hasPopulationCount()) 1810c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::CTPOP, VT, Custom); 1820c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else 1830c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::CTPOP, VT, Expand); 1840c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 1851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // No special instructions for these. 1861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::CTTZ, VT, Expand); 1871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ROTR, VT, Expand); 1881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 189df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // Use *MUL_LOHI where possible instead of MULH*. 1901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::MULHS, VT, Expand); 1911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::MULHU, VT, Expand); 192df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford setOperationAction(ISD::SMUL_LOHI, VT, Custom); 193df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford setOperationAction(ISD::UMUL_LOHI, VT, Custom); 1941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 19536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only z196 and above have native support for conversions to unsigned. 19636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Subtarget.hasFPExtension()) 19736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::FP_TO_UINT, VT, Expand); 1981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 1991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 2001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Type legalization will convert 8- and 16-bit atomic operations into 2021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // forms that operate on i32s (but still keeping the original memory VT). 2031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Lower them into full i32 operations. 2041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 2051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 2061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 2071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 2081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 2091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 2101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 2111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 2121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 2131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 2141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 2151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 2161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 217de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); 218de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 219de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Traps are legal, as we will convert them to "j .+2". 220de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::TRAP, MVT::Other, Legal); 221de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 22236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // z10 has instructions for signed but not unsigned FP conversion. 2231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Handle unsigned 32-bit types as signed 64-bit types. 22436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Subtarget.hasFPExtension()) { 22536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 22636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 22736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 2281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We have native support for a 64-bit CTLZ, via FLOGR. 2301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::CTLZ, MVT::i32, Promote); 2311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::CTLZ, MVT::i64, Legal); 2321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Give LowerOperation the chance to replace 64-bit ORs with subregs. 2341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::OR, MVT::i64, Custom); 2351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // FIXME: Can we support these natively? 2371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 2381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 2391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 2401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We have native instructions for i8, i16 and i32 extensions, but not i1. 2421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 243ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::integer_valuetypes()) { 244ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 245ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 246ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 247ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Handle the various types of symbolic address. 2501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::ConstantPool, PtrVT, Custom); 2511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 2521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 2531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::BlockAddress, PtrVT, Custom); 2541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::JumpTable, PtrVT, Custom); 2551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We need to handle dynamic allocations specially because of the 2571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // 160-byte area at the bottom of the stack. 2581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 259de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, PtrVT, Custom); 2601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Use custom expanders so that we can force the function to use 2621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // a frame pointer. 2631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 2641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 2651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 266a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford // Handle prefetches with PFD or PFDRL. 267a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 268a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford 2696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (MVT VT : MVT::vector_valuetypes()) { 2706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Assume by default that all vector operations need to be expanded. 2716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode) 2726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (getOperationAction(Opcode, VT) == Legal) 2736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(Opcode, VT, Expand); 2746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 2756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Likewise all truncating stores and extending loads. 2766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (MVT InnerVT : MVT::vector_valuetypes()) { 2776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setTruncStoreAction(VT, InnerVT, Expand); 2786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 2796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 2806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 2816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 2826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 2836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isTypeLegal(VT)) { 2846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // These operations are legal for anything that can be stored in a 2856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // vector register, even if there is no native support for the format 2866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // as such. In particular, we can do these for v4f32 even though there 2876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // are no specific instructions for that format. 2886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::LOAD, VT, Legal); 2896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::STORE, VT, Legal); 2906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::VSELECT, VT, Legal); 2916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::BITCAST, VT, Legal); 2926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::UNDEF, VT, Legal); 2936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 2946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Likewise, except that we need to replace the nodes with something 2956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // more specific. 2966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 2976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 2986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 2996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle integer vector types. 3026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (MVT VT : MVT::integer_vector_valuetypes()) { 3036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isTypeLegal(VT)) { 3046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // These operations have direct equivalents. 3056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); 3066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal); 3076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::ADD, VT, Legal); 3086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SUB, VT, Legal); 3096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT != MVT::v2i64) 3106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::MUL, VT, Legal); 3116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::AND, VT, Legal); 3126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::OR, VT, Legal); 3136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::XOR, VT, Legal); 3146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::CTPOP, VT, Custom); 3156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::CTTZ, VT, Legal); 3166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::CTLZ, VT, Legal); 3176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Convert a GPR scalar to a vector by inserting it into element 0. 3196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); 3206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Use a series of unpacks for extensions. 3226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); 3236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); 3246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Detect shifts by a scalar amount and convert them into 3266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // V*_BY_SCALAR. 3276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SHL, VT, Custom); 3286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SRA, VT, Custom); 3296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SRL, VT, Custom); 3306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // At present ROTL isn't matched by DAGCombiner. ROTR should be 3326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // converted into ROTL. 3336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::ROTL, VT, Expand); 3346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::ROTR, VT, Expand); 3356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands 3376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // and inverting the result as necessary. 3386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SETCC, VT, Custom); 3396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 3436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // There should be no need to check for float types other than v2f64 3446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // since <2 x f32> isn't a legal type. 3456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 3466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 3476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 3486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 3496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Handle floating-point types. 3521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = MVT::FIRST_FP_VALUETYPE; 3531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand I <= MVT::LAST_FP_VALUETYPE; 3541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ++I) { 3551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MVT VT = MVT::SimpleValueType(I); 3561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isTypeLegal(VT)) { 3571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We can use FI for FRINT. 3581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FRINT, VT, Legal); 3591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 360d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford // We can use the extended form of FI for other rounding operations. 361d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford if (Subtarget.hasFPExtension()) { 362d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford setOperationAction(ISD::FNEARBYINT, VT, Legal); 363d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford setOperationAction(ISD::FFLOOR, VT, Legal); 364d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford setOperationAction(ISD::FCEIL, VT, Legal); 365d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford setOperationAction(ISD::FTRUNC, VT, Legal); 366d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford setOperationAction(ISD::FROUND, VT, Legal); 367d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford } 368d95865a2a2daeb7dd8b80c18e7409b28e7e4738aRichard Sandiford 3691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // No special instructions for these. 3701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FSIN, VT, Expand); 3711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FCOS, VT, Expand); 372f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FSINCOS, VT, Expand); 3731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FREM, VT, Expand); 374f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FPOW, VT, Expand); 3751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 3761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 3771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 3786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle floating-point vector types. 3796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 3806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Scalar-to-vector conversion is just a subreg. 3816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 3826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 3836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Some insertions and extractions can be done directly but others 3856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // need to go via integers. 3866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 3876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 3886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 3896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 3906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 3916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // These operations have direct equivalents. 3926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FADD, MVT::v2f64, Legal); 3936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FNEG, MVT::v2f64, Legal); 3946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 3956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 3966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FMA, MVT::v2f64, Legal); 3976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 3986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FABS, MVT::v2f64, Legal); 3996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 4006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FRINT, MVT::v2f64, Legal); 4016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 4026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 4036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 4046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 4056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 4066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 4081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We have fused multiply-addition for f32 and f64 but not f128. 4091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FMA, MVT::f32, Legal); 4101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FMA, MVT::f64, Legal); 4111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::FMA, MVT::f128, Expand); 4121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 4131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Needed so that we don't try to implement f128 constant loads using 4141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // a load-and-extend of a f80 constant (in cases where the constant 4151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // would fit in an f80). 416ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::fp_valuetypes()) 417ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); 4181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 4191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Floating-point truncation and stores need to be done separately. 4201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setTruncStoreAction(MVT::f64, MVT::f32, Expand); 4211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setTruncStoreAction(MVT::f128, MVT::f32, Expand); 4221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setTruncStoreAction(MVT::f128, MVT::f64, Expand); 4231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 4241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We have 64-bit FPR<->GPR moves, but need special handling for 4251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // 32-bit forms. 4266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Subtarget.hasVector()) { 4276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::BITCAST, MVT::i32, Custom); 4286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::BITCAST, MVT::f32, Custom); 4296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 4311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // VASTART and VACOPY need to deal with the SystemZ-specific varargs 4321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // structure, but VAEND is a no-op. 4331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::VASTART, MVT::Other, Custom); 4341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::VACOPY, MVT::Other, Custom); 4351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand setOperationAction(ISD::VAEND, MVT::Other, Expand); 436dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Codes for which we want to perform some z-specific combinations. 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setTargetDAGCombine(ISD::SIGN_EXTEND); 4396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setTargetDAGCombine(ISD::STORE); 4406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 4416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setTargetDAGCombine(ISD::FP_ROUND); 442de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::BSWAP); 443de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::SHL); 444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::SRA); 445de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::SRL); 446de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::ROTL); 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4480c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Handle intrinsics. 4490c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 4506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 4510c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 452dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford // We want to use MVC in preference to even a single load/store pair. 453dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford MaxStoresPerMemcpy = 0; 454dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford MaxStoresPerMemcpyOptSize = 0; 455f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford 456f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford // The main memset sequence is a byte store followed by an MVC. 457f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford // Two STC or MV..I stores win over that, but the kind of fused stores 458f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford // generated by target-independent code don't when the byte value is 459f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 460f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford // than "STC;MVC". Handle the choice in target-specific code instead. 461f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford MaxStoresPerMemset = 0; 462f6ea5e0d8007234fc74c1ff6ac2c3ca316c41d92Richard Sandiford MaxStoresPerMemsetOptSize = 0; 4631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 4641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 465f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarEVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL, 466f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LLVMContext &, EVT VT) const { 467d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandiford if (!VT.isVector()) 468d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandiford return MVT::i32; 469d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandiford return VT.changeVectorElementTypeToInteger(); 470d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandiford} 471d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandiford 472d975a82e08ab15242724bc46e53d58b4ff3e4815Richard Sandifordbool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 473e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin VT = VT.getScalarType(); 474e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 475e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin if (!VT.isSimple()) 476e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin return false; 477e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 478e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin switch (VT.getSimpleVT().SimpleTy) { 479e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin case MVT::f32: 480e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin case MVT::f64: 481e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin return true; 482e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin case MVT::f128: 483e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin return false; 484e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin default: 485e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin break; 486e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin } 487e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 488e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin return false; 489e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin} 490e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 4911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandbool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 4921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 4931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Imm.isZero() || Imm.isNegZero(); 4941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 4951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 4960c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarbool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 4970c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // We can use CGFI or CLGFI. 4980c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return isInt<32>(Imm) || isUInt<32>(Imm); 4990c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 5000c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 5010c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarbool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const { 5020c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // We can use ALGFI or SLGFI. 5030c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return isUInt<32>(Imm) || isUInt<32>(-Imm); 5040c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 5050c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 50637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 50737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned, 50837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned, 50937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool *Fast) const { 51014a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford // Unaligned accesses should never be slower than the expanded version. 51114a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford // We check specifically for aligned accesses in the few cases where 51214a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford // they are required. 51314a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford if (Fast) 51414a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford *Fast = true; 51514a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford return true; 51614a926f13b768ee3771bb944bbbb29529a40dbe1Richard Sandiford} 5176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 518f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL, 519f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AddrMode &AM, Type *Ty, 5206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned AS) const { 52104ded924f3583438c6633823eddb87761fa73cceRichard Sandiford // Punt on globals for now, although they can be used in limited 52204ded924f3583438c6633823eddb87761fa73cceRichard Sandiford // RELATIVE LONG cases. 52304ded924f3583438c6633823eddb87761fa73cceRichard Sandiford if (AM.BaseGV) 52404ded924f3583438c6633823eddb87761fa73cceRichard Sandiford return false; 52504ded924f3583438c6633823eddb87761fa73cceRichard Sandiford 52604ded924f3583438c6633823eddb87761fa73cceRichard Sandiford // Require a 20-bit signed offset. 52704ded924f3583438c6633823eddb87761fa73cceRichard Sandiford if (!isInt<20>(AM.BaseOffs)) 52804ded924f3583438c6633823eddb87761fa73cceRichard Sandiford return false; 52904ded924f3583438c6633823eddb87761fa73cceRichard Sandiford 53004ded924f3583438c6633823eddb87761fa73cceRichard Sandiford // Indexing is OK but no scale factor can be applied. 53104ded924f3583438c6633823eddb87761fa73cceRichard Sandiford return AM.Scale == 0 || AM.Scale == 1; 53204ded924f3583438c6633823eddb87761fa73cceRichard Sandiford} 53304ded924f3583438c6633823eddb87761fa73cceRichard Sandiford 53480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandifordbool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 53580f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 53680f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return false; 53780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford unsigned FromBits = FromType->getPrimitiveSizeInBits(); 53880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford unsigned ToBits = ToType->getPrimitiveSizeInBits(); 53980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return FromBits > ToBits; 54080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford} 54180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 54280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandifordbool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 54380f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (!FromVT.isInteger() || !ToVT.isInteger()) 54480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return false; 54580f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford unsigned FromBits = FromVT.getSizeInBits(); 54680f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford unsigned ToBits = ToVT.getSizeInBits(); 54780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return FromBits > ToBits; 54880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford} 54980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 5501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 5511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Inline asm support 5521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 5531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandTargetLowering::ConstraintType 555f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarSystemZTargetLowering::getConstraintType(StringRef Constraint) const { 5561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Constraint.size() == 1) { 5571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (Constraint[0]) { 5581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'a': // Address register 5591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'd': // Data register (equivalent to 'r') 5601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'f': // Floating-point register 56155d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford case 'h': // High-part register 5621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'r': // General-purpose register 5631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return C_RegisterClass; 5641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'Q': // Memory with base and unsigned 12-bit displacement 5661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'R': // Likewise, plus an index 5671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'S': // Memory with base and signed 20-bit displacement 5681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'T': // Likewise, plus an index 5691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'm': // Equivalent to 'T'. 5701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return C_Memory; 5711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'I': // Unsigned 8-bit constant 5731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'J': // Unsigned 12-bit constant 5741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'K': // Signed 16-bit constant 5751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'L': // Signed 20-bit displacement (on all targets we support) 5761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'M': // 0x7fffffff 5771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return C_Other; 5781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 5801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 5811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 5821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 5831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return TargetLowering::getConstraintType(Constraint); 5841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 5851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandTargetLowering::ConstraintWeight SystemZTargetLowering:: 5871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandgetSingleConstraintMatchWeight(AsmOperandInfo &info, 5881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const char *constraint) const { 5891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ConstraintWeight weight = CW_Invalid; 5901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Value *CallOperandVal = info.CallOperandVal; 5911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // If we don't have a value, we can't do a match, 5921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // but allow it at the lowest weight. 593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CallOperandVal) 5941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return CW_Default; 5951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Type *type = CallOperandVal->getType(); 5961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Look at the constraint type. 5971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (*constraint) { 5981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 5991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 6001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'a': // Address register 6031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'd': // Data register (equivalent to 'r') 60455d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford case 'h': // High-part register 6051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'r': // General-purpose register 6061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (CallOperandVal->getType()->isIntegerTy()) 6071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Register; 6081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'f': // Floating-point register 6111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (type->isFloatingPointTy()) 6121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Register; 6131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'I': // Unsigned 8-bit constant 61636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 6171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isUInt<8>(C->getZExtValue())) 6181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Constant; 6191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'J': // Unsigned 12-bit constant 62236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 6231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isUInt<12>(C->getZExtValue())) 6241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Constant; 6251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'K': // Signed 16-bit constant 62836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 6291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isInt<16>(C->getSExtValue())) 6301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Constant; 6311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'L': // Signed 20-bit displacement (on all targets we support) 63436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 6351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isInt<20>(C->getSExtValue())) 6361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Constant; 6371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'M': // 0x7fffffff 64036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 6411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (C->getZExtValue() == 0x7fffffff) 6421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand weight = CW_Constant; 6431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 6441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 6451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return weight; 6461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 6471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6485e009541973b7935386055066689902aa7134e2dRichard Sandiford// Parse a "{tNNN}" register constraint for which the register type "t" 6495e009541973b7935386055066689902aa7134e2dRichard Sandiford// has already been verified. MC is the class associated with "t" and 6505e009541973b7935386055066689902aa7134e2dRichard Sandiford// Map maps 0-based register numbers to LLVM register numbers. 6515e009541973b7935386055066689902aa7134e2dRichard Sandifordstatic std::pair<unsigned, const TargetRegisterClass *> 652f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarparseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC, 653f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned *Map) { 6545e009541973b7935386055066689902aa7134e2dRichard Sandiford assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 6555e009541973b7935386055066689902aa7134e2dRichard Sandiford if (isdigit(Constraint[2])) { 656f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Index; 657f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool Failed = 658f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index); 659f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Failed && Index < 16 && Map[Index]) 6605e009541973b7935386055066689902aa7134e2dRichard Sandiford return std::make_pair(Map[Index], RC); 6615e009541973b7935386055066689902aa7134e2dRichard Sandiford } 662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, nullptr); 6635e009541973b7935386055066689902aa7134e2dRichard Sandiford} 6645e009541973b7935386055066689902aa7134e2dRichard Sandiford 665ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstd::pair<unsigned, const TargetRegisterClass *> 666ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesSystemZTargetLowering::getRegForInlineAsmConstraint( 667f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 6681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Constraint.size() == 1) { 6691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // GCC Constraint Letters 6701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (Constraint[0]) { 6711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: break; 6721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'd': // Data register (equivalent to 'r') 6731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'r': // General-purpose register 6741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VT == MVT::i64) 6751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::GR64BitRegClass); 6761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if (VT == MVT::i128) 6771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::GR128BitRegClass); 6781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::GR32BitRegClass); 6791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 6801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'a': // Address register 6811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VT == MVT::i64) 6821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 6831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if (VT == MVT::i128) 6841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 6851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 6861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 68755d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford case 'h': // High-part register (an LLVM extension) 68855d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 68955d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford 6901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'f': // Floating-point register 6911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VT == MVT::f64) 6921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::FP64BitRegClass); 6931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if (VT == MVT::f128) 6941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::FP128BitRegClass); 6951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return std::make_pair(0U, &SystemZ::FP32BitRegClass); 6961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 6971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 698f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Constraint.size() > 0 && Constraint[0] == '{') { 6995e009541973b7935386055066689902aa7134e2dRichard Sandiford // We need to override the default register parsing for GPRs and FPRs 7005e009541973b7935386055066689902aa7134e2dRichard Sandiford // because the interpretation depends on VT. The internal names of 7015e009541973b7935386055066689902aa7134e2dRichard Sandiford // the registers are also different from the external names 7025e009541973b7935386055066689902aa7134e2dRichard Sandiford // (F0D and F0S instead of F0, etc.). 7035e009541973b7935386055066689902aa7134e2dRichard Sandiford if (Constraint[1] == 'r') { 7045e009541973b7935386055066689902aa7134e2dRichard Sandiford if (VT == MVT::i32) 7055e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 7065e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::GR32Regs); 7075e009541973b7935386055066689902aa7134e2dRichard Sandiford if (VT == MVT::i128) 7085e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 7095e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::GR128Regs); 7105e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 7115e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::GR64Regs); 7125e009541973b7935386055066689902aa7134e2dRichard Sandiford } 7135e009541973b7935386055066689902aa7134e2dRichard Sandiford if (Constraint[1] == 'f') { 7145e009541973b7935386055066689902aa7134e2dRichard Sandiford if (VT == MVT::f32) 7155e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 7165e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::FP32Regs); 7175e009541973b7935386055066689902aa7134e2dRichard Sandiford if (VT == MVT::f128) 7185e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 7195e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::FP128Regs); 7205e009541973b7935386055066689902aa7134e2dRichard Sandiford return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 7215e009541973b7935386055066689902aa7134e2dRichard Sandiford SystemZMC::FP64Regs); 7225e009541973b7935386055066689902aa7134e2dRichard Sandiford } 7235e009541973b7935386055066689902aa7134e2dRichard Sandiford } 724ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 7251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 7261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandvoid SystemZTargetLowering:: 7281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandLowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 7291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand std::vector<SDValue> &Ops, 7301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 7311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Only support length 1 constraints for now. 7321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Constraint.length() == 1) { 7331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (Constraint[0]) { 7341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'I': // Unsigned 8-bit constant 73536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantSDNode>(Op)) 7361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isUInt<8>(C->getZExtValue())) 7376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 7381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getValueType())); 7391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 7401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'J': // Unsigned 12-bit constant 74236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantSDNode>(Op)) 7431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isUInt<12>(C->getZExtValue())) 7446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 7451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getValueType())); 7461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 7471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'K': // Signed 16-bit constant 74936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantSDNode>(Op)) 7501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isInt<16>(C->getSExtValue())) 7516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), 7521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getValueType())); 7531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 7541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'L': // Signed 20-bit displacement (on all targets we support) 75636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantSDNode>(Op)) 7571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (isInt<20>(C->getSExtValue())) 7586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), 7591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getValueType())); 7601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 7611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case 'M': // 0x7fffffff 76336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *C = dyn_cast<ConstantSDNode>(Op)) 7641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (C->getZExtValue() == 0x7fffffff) 7656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 7661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getValueType())); 7671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 7681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 7691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 7701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 7711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 7721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 7741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Calling conventions 7751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 7761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 7771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#include "SystemZGenCallingConv.inc" 7781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 77980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandifordbool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 78080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Type *ToType) const { 78180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return isTruncateFree(FromType, ToType); 78280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford} 78380f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 78480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandifordbool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 785f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return CI->isTailCall(); 78680f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford} 78780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 7886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// We do not yet support 128-bit single-element vector types. If the user 7896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// attempts to use such types as function argument or return type, prefer 7906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// to error out instead of emitting code violating the ABI. 7916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic void VerifyVectorType(MVT VT, EVT ArgVT) { 7926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (ArgVT.isVector() && !VT.isVector()) 7936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar report_fatal_error("Unsupported vector argument or return type"); 7946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 7956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 7966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic void VerifyVectorTypes(const SmallVectorImpl<ISD::InputArg> &Ins) { 7976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned i = 0; i < Ins.size(); ++i) 7986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorType(Ins[i].VT, Ins[i].ArgVT); 7996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 8006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic void VerifyVectorTypes(const SmallVectorImpl<ISD::OutputArg> &Outs) { 8026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned i = 0; i < Outs.size(); ++i) 8036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorType(Outs[i].VT, Outs[i].ArgVT); 8046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 8056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Value is a value that has been passed to us in the location described by VA 8071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 8081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// any loads onto Chain. 809de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL, 8101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA, SDValue Chain, 8111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Value) { 8121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // If the argument has been promoted from a smaller type, insert an 8131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // assertion to capture this. 8141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.getLocInfo() == CCValAssign::SExt) 8151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 8161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand DAG.getValueType(VA.getValVT())); 8171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if (VA.getLocInfo() == CCValAssign::ZExt) 8181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 8191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand DAG.getValueType(VA.getValVT())); 8201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 8211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.isExtInLoc()) 8221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 8236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (VA.getLocInfo() == CCValAssign::BCvt) { 8246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If this is a short vector argument loaded from the stack, 8256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // extend from i64 to full vector size and then bitcast. 8266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(VA.getLocVT() == MVT::i64); 8276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(VA.getValVT().isVector()); 828de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)}); 8296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value); 8306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else 8311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 8321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Value; 8331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 8341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 8351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Value is a value of type VA.getValVT() that we need to copy into 8361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// the location described by VA. Return a copy of Value converted to 8371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// VA.getValVT(). The caller is responsible for handling indirect values. 838de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL, 8391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA, SDValue Value) { 8401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (VA.getLocInfo()) { 8411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case CCValAssign::SExt: 8421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 8431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case CCValAssign::ZExt: 8441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 8451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case CCValAssign::AExt: 8461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 8476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case CCValAssign::BCvt: 8486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If this is a short vector argument to be stored to the stack, 8496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // bitcast to v2i64 and then extract first element. 8506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(VA.getLocVT() == MVT::i64); 8516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(VA.getValVT().isVector()); 8526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value); 8536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value, 8546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)); 8551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case CCValAssign::Full: 8561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Value; 8571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 8581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unhandled getLocInfo()"); 8591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 8601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 8611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 862de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::LowerFormalArguments( 863de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 864de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 865de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 8661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 8671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFrameInfo *MFI = MF.getFrameInfo(); 8681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineRegisterInfo &MRI = MF.getRegInfo(); 8691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZMachineFunctionInfo *FuncInfo = 870ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MF.getInfo<SystemZMachineFunctionInfo>(); 871ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *TFL = 872ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZFrameLowering *>(Subtarget.getFrameLowering()); 873de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 8741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 8756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Detect unsupported vector argument types. 8766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) 8776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorTypes(Ins); 8786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Assign locations to all of the incoming arguments. 8801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<CCValAssign, 16> ArgLocs; 8816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 8821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 8831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 8841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NumFixedGPRs = 0; 8851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NumFixedFPRs = 0; 8861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 8871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ArgValue; 8881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA = ArgLocs[I]; 8891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT LocVT = VA.getLocVT(); 8901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.isRegLoc()) { 8911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Arguments passed in registers 8921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const TargetRegisterClass *RC; 8931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (LocVT.getSimpleVT().SimpleTy) { 8941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 8951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Integers smaller than i64 should be promoted to i64. 8961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unexpected argument type"); 8971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::i32: 8981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NumFixedGPRs += 1; 8991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RC = &SystemZ::GR32BitRegClass; 9001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 9011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::i64: 9021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NumFixedGPRs += 1; 9031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RC = &SystemZ::GR64BitRegClass; 9041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 9051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::f32: 9061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NumFixedFPRs += 1; 9071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RC = &SystemZ::FP32BitRegClass; 9081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 9091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case MVT::f64: 9101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NumFixedFPRs += 1; 9111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RC = &SystemZ::FP64BitRegClass; 9121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand break; 9136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v16i8: 9146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v8i16: 9156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v4i32: 9166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v2i64: 9176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v4f32: 9186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case MVT::v2f64: 9196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RC = &SystemZ::VR128BitRegClass; 9206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 9211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 9221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned VReg = MRI.createVirtualRegister(RC); 9241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MRI.addLiveIn(VA.getLocReg(), VReg); 9251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 9261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else { 9271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(VA.isMemLoc() && "Argument not register or memory"); 9281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Create the frame index object for this incoming parameter. 9301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 9311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand VA.getLocMemOffset(), true); 9321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Create the SelectionDAG nodes corresponding to a load 9341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // from this parameter. Unpromoted ints and floats are 9351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // passed as right-justified 8-byte values. 9361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 9371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 9386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, 9396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(4, DL)); 9401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 941f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getFixedStack(MF, FI), false, 942f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 9431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 9441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Convert the value of the argument register into the value that's 9461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // being passed. 947de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VA.getLocInfo() == CCValAssign::Indirect) { 948de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, 949de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ArgValue, MachinePointerInfo(), 950de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, false, 0)); 951de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the original argument was split (e.g. i128), we need 952de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // to load all parts of it here (using the same address). 953de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ArgIndex = Ins[I].OrigArgIndex; 954de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert (Ins[I].PartOffset == 0); 955de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) { 956de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCValAssign &PartVA = ArgLocs[I + 1]; 957de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned PartOffset = Ins[I + 1].PartOffset; 958de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, 959de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getIntPtrConstant(PartOffset, DL)); 960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, 961de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Address, MachinePointerInfo(), 962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, false, 0)); 963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++I; 964de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else 966de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 9671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 9681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsVarArg) { 9701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Save the number of non-varargs registers for later use by va_start, etc. 9711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 9721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 9731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Likewise the address (in the form of a frame index) of where the 9751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // first stack vararg would be. The 1-byte size here is arbitrary. 9761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t StackSize = CCInfo.getNextStackOffset(); 9771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 9781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ...and a similar frame index for the caller-allocated save area 9801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // that will be used to store the incoming registers. 9811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 9821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 9831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 9841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 9851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Store the FPR varargs in the reserved frame slots. (We store the 9861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // GPRs as part of the prologue.) 9871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (NumFixedFPRs < SystemZ::NumArgFPRs) { 9881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue MemOps[SystemZ::NumArgFPRs]; 9891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 9901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 9911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 992f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 9931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 9941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand &SystemZ::FP64BitRegClass); 9951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 9961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 997f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getFixedStack(MF, FI), 9981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand false, false, 0); 9991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 10001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Join the stores, which are independent of one another. 10011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 1002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines makeArrayRef(&MemOps[NumFixedFPRs], 1003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SystemZ::NumArgFPRs-NumFixedFPRs)); 10041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 10051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 10061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 10071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Chain; 10081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 10091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 101037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool canUseSiblingCall(const CCState &ArgCCInfo, 1011de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SmallVectorImpl<CCValAssign> &ArgLocs, 1012de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SmallVectorImpl<ISD::OutputArg> &Outs) { 101380f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // Punt if there are any indirect or stack arguments, or if the call 1014de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // needs the callee-saved argument register R6, or if the call uses 1015de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // the callee-saved register arguments SwiftSelf and SwiftError. 101680f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 101780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford CCValAssign &VA = ArgLocs[I]; 101880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (VA.getLocInfo() == CCValAssign::Indirect) 101980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return false; 102080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (!VA.isRegLoc()) 102180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return false; 102280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford unsigned Reg = VA.getLocReg(); 102355d7d83b6c9e55fa73d667660c8e90f92999385bRichard Sandiford if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 102480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return false; 1025de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError()) 1026de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 102780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford } 102880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford return true; 102980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford} 103080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 10311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue 10321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 10331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVectorImpl<SDValue> &InVals) const { 10341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG = CLI.DAG; 1035ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc &DL = CLI.DL; 1036a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Topper SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1037a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Topper SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1038a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Topper SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 10391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Chain = CLI.Chain; 10401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Callee = CLI.Callee; 104180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford bool &IsTailCall = CLI.IsTailCall; 10421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CallingConv::ID CallConv = CLI.CallConv; 10431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand bool IsVarArg = CLI.IsVarArg; 10441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 1045f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(MF.getDataLayout()); 10461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 10476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Detect unsupported vector argument and return types. 10486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 10496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorTypes(Outs); 10506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorTypes(Ins); 10516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 10526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 10531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Analyze the operands of the call, assigning locations to each operand. 10541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<CCValAssign, 16> ArgLocs; 10556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 10561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 10571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 105880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // We don't support GuaranteedTailCallOpt, only automatically-detected 105980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // sibling calls. 1060de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs)) 106180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford IsTailCall = false; 106280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 10631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get a count of how many bytes are to be pushed on the stack. 10641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 10651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 10661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Mark the start of the call. 106780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (!IsTailCall) 10686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Chain = DAG.getCALLSEQ_START(Chain, 10696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumBytes, DL, PtrVT, true), 107080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford DL); 10711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 10721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Copy argument values to their designated locations. 10731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 10741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<SDValue, 8> MemOpChains; 10751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue StackPtr; 10761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 10771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA = ArgLocs[I]; 10781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ArgValue = OutVals[I]; 10791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 10801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.getLocInfo() == CCValAssign::Indirect) { 10811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Store the argument in a stack slot and pass its address. 1082de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT); 10831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 1084f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MemOpChains.push_back(DAG.getStore( 1085f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Chain, DL, ArgValue, SpillSlot, 1086f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getFixedStack(MF, FI), false, false, 0)); 1087de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the original argument was split (e.g. i128), we need 1088de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // to store all parts of it here (and pass just one address). 1089de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ArgIndex = Outs[I].OrigArgIndex; 1090de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert (Outs[I].PartOffset == 0); 1091de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { 1092de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue PartValue = OutVals[I + 1]; 1093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned PartOffset = Outs[I + 1].PartOffset; 1094de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, 1095de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getIntPtrConstant(PartOffset, DL)); 1096de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MemOpChains.push_back(DAG.getStore( 1097de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Chain, DL, PartValue, Address, 1098de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachinePointerInfo::getFixedStack(MF, FI), false, false, 0)); 1099de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ++I; 1100de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 11011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ArgValue = SpillSlot; 11021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else 11031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 11041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.isRegLoc()) 11061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Queue up the argument copies and emit them at the end. 11071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 11081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else { 11091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(VA.isMemLoc() && "Argument not register or memory"); 11101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Work out the address of the stack slot. Unpromoted ints and 11121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // floats are passed as right-justified 8-byte values. 11131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (!StackPtr.getNode()) 11141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 11151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 11161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 11171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Offset += 4; 11181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 11196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(Offset, DL)); 11201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Emit the store. 11221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 11231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachinePointerInfo(), 11241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand false, false, 0)); 11251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 11261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 11271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Join the stores, which are independent of one another. 11291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (!MemOpChains.empty()) 1130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 11311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Accept direct calls by converting symbolic call addresses to the 113380f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // associated Target* opcodes. Force %r1 to be used for indirect 113480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // tail calls. 113580f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford SDValue Glue; 113636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 11371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 11381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 113936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 11401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 11411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 114280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford } else if (IsTailCall) { 114380f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 114480f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Glue = Chain.getValue(1); 114580f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 114680f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford } 114780f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford 114880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford // Build a sequence of copy-to-reg nodes, chained and glued together. 114980f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 115080f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 115180f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford RegsToPass[I].second, Glue); 115280f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford Glue = Chain.getValue(1); 11531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 11541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The first call operand is the chain and the second is the target address. 11561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<SDValue, 8> Ops; 11571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Ops.push_back(Chain); 11581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Ops.push_back(Callee); 11591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Add argument registers to the end of the list so that they are 11611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // known live into the call. 11621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 11631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Ops.push_back(DAG.getRegister(RegsToPass[I].first, 11641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RegsToPass[I].second.getValueType())); 11651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1166c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Add a register mask operand representing the call-preserved registers. 1167ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 11684c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); 1169c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines assert(Mask && "Missing call preserved mask for calling convention"); 1170c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Ops.push_back(DAG.getRegisterMask(Mask)); 1171c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 11721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Glue the call to the argument copies, if any. 11731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Glue.getNode()) 11741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Ops.push_back(Glue); 11751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Emit the call. 11771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 117880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford if (IsTailCall) 1179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops); 1180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops); 11811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Glue = Chain.getValue(1); 11821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Mark the end of the call, which is glued to the call itself. 11841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Chain = DAG.getCALLSEQ_END(Chain, 11856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumBytes, DL, PtrVT, true), 11866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, PtrVT, true), 11876e0b2a0cb0d398f175a5294bf0ad5488c714e8c2Andrew Trick Glue, DL); 11881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Glue = Chain.getValue(1); 11891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Assign locations to each value returned by this call. 11911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<CCValAssign, 16> RetLocs; 119237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 11931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 11941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Copy all of the result registers out of their specified physreg. 11961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 11971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA = RetLocs[I]; 11981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 11991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Copy the value out, gluing the copy to the end of the call sequence. 12001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 12011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand VA.getLocVT(), Glue); 12021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Chain = RetValue.getValue(1); 12031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Glue = RetValue.getValue(2); 12041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Convert the value of the return register into the value that's 12061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // being returned. 12071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 12081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 12091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Chain; 12111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 12121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1213f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool SystemZTargetLowering:: 1214f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarCanLowerReturn(CallingConv::ID CallConv, 1215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineFunction &MF, bool isVarArg, 1216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const SmallVectorImpl<ISD::OutputArg> &Outs, 1217f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LLVMContext &Context) const { 1218f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Detect unsupported vector return types. 1219f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Subtarget.hasVector()) 1220f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VerifyVectorTypes(Outs); 1221f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1222de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Special case that we cannot easily detect in RetCC_SystemZ since 1223de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // i128 is not a legal type. 1224de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar for (auto &Out : Outs) 1225de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Out.ArgVT == MVT::i128) 1226de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 1227de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallVector<CCValAssign, 16> RetLocs; 1229f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context); 1230f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ); 1231f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 1232f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 12331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue 1234de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 1235de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool IsVarArg, 12361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const SmallVectorImpl<ISD::OutputArg> &Outs, 12371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const SmallVectorImpl<SDValue> &OutVals, 1238de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, SelectionDAG &DAG) const { 12391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 12401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Detect unsupported vector return types. 12426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) 12436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VerifyVectorTypes(Outs); 12446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 12451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Assign locations to each returned value. 12461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<CCValAssign, 16> RetLocs; 124737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 12481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 12491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Quick exit for void returns 12511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (RetLocs.empty()) 12521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 12531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Copy the result values into the output registers. 12551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Glue; 12561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SmallVector<SDValue, 4> RetOps; 12571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetOps.push_back(Chain); 12581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 12591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CCValAssign &VA = RetLocs[I]; 12601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue RetValue = OutVals[I]; 12611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Make the return register live on exit. 12631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(VA.isRegLoc() && "Can only return in registers!"); 12641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Promote the value as required. 12661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 12671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Chain and glue the copies together. 12691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Reg = VA.getLocReg(); 12701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 12711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Glue = Chain.getValue(1); 12721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 12731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 12741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 12751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Update chain and glue. 12761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetOps[0] = Chain; 12771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Glue.getNode()) 12781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand RetOps.push_back(Glue); 12791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps); 12811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 12821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1283de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::prepareVolatileOrAtomicLoad( 1284de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const { 128536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(SystemZISD::SERIALIZE, DL, MVT::Other, Chain); 128636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 128736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Return true if Op is an intrinsic node with chain that returns the CC value 12890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// as its only (other) argument. Provide the associated SystemZISD opcode and 12900c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// the mask of valid CC values if so. 12910c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarstatic bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode, 12920c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned &CCValid) { 12930c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 12940c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar switch (Id) { 12950c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Intrinsic::s390_tbegin: 12960c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Opcode = SystemZISD::TBEGIN; 12970c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_TBEGIN; 12980c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return true; 12990c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 13000c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Intrinsic::s390_tbegin_nofloat: 13010c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Opcode = SystemZISD::TBEGIN_NOFLOAT; 13020c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_TBEGIN; 13030c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return true; 13040c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 13050c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Intrinsic::s390_tend: 13060c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Opcode = SystemZISD::TEND; 13070c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_TEND; 13080c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return true; 13090c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 13100c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar default: 13110c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 13120c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 13130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 13140c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 13156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return true if Op is an intrinsic node without chain that returns the 13166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// CC value as its final argument. Provide the associated SystemZISD 13176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// opcode and the mask of valid CC values if so. 13186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) { 13196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 13206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch (Id) { 13216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpkshs: 13226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpksfs: 13236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpksgs: 13246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::PACKS_CC; 13256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpklshs: 13296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpklsfs: 13306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpklsgs: 13316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::PACKLS_CC; 13326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vceqbs: 13366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vceqhs: 13376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vceqfs: 13386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vceqgs: 13396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VICMPES; 13406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchbs: 13446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchhs: 13456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchfs: 13466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchgs: 13476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VICMPHS; 13486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchlbs: 13526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchlhs: 13536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchlfs: 13546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vchlgs: 13556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VICMPHLS; 13566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vtm: 13606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VTM; 13616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 13626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaebs: 13656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaehs: 13666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaefs: 13676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFAE_CC; 13686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 13696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaezbs: 13726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaezhs: 13736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfaezfs: 13746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFAEZ_CC; 13756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 13766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeebs: 13796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeehs: 13806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeefs: 13816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFEE_CC; 13826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 13836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeezbs: 13866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeezhs: 13876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfeezfs: 13886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFEEZ_CC; 13896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 13906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenebs: 13936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenehs: 13946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenefs: 13956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFENE_CC; 13966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 13976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 13986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenezbs: 14006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenezhs: 14016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfenezfs: 14026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFENEZ_CC; 14036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 14046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vistrbs: 14076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vistrhs: 14086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vistrfs: 14096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VISTR_CC; 14106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3; 14116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrcbs: 14146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrchs: 14156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrcfs: 14166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VSTRC_CC; 14176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 14186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrczbs: 14216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrczhs: 14226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vstrczfs: 14236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VSTRCZ_CC; 14246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_ANY; 14256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfcedbs: 14286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFCMPES; 14296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 14306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfchdbs: 14336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFCMPHS; 14346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 14356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vfchedbs: 14386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFCMPHES; 14396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 14406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vftcidb: 14436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = SystemZISD::VFTCI; 14446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCValid = SystemZ::CCMASK_VCMP; 14456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 14466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1447de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case Intrinsic::s390_tdc: 1448de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Opcode = SystemZISD::TDC; 1449de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCValid = SystemZ::CCMASK_TDC; 1450de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 1451de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 14526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar default: 14536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 14546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 14556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 14566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14570c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Emit an intrinsic with chain with a glued value instead of its CC result. 14580c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarstatic SDValue emitIntrinsicWithChainAndGlue(SelectionDAG &DAG, SDValue Op, 14590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Opcode) { 14600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Copy all operands except the intrinsic ID. 14610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned NumOps = Op.getNumOperands(); 14620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SmallVector<SDValue, 6> Ops; 14630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Ops.reserve(NumOps - 1); 14640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Ops.push_back(Op.getOperand(0)); 14650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar for (unsigned I = 2; I < NumOps; ++I) 14660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Ops.push_back(Op.getOperand(I)); 14670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 14680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar assert(Op->getNumValues() == 2 && "Expected only CC result and chain"); 14690c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDVTList RawVTs = DAG.getVTList(MVT::Other, MVT::Glue); 14700c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops); 14710c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue OldChain = SDValue(Op.getNode(), 1); 14720c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue NewChain = SDValue(Intr.getNode(), 0); 14730c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain); 14740c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return Intr; 14750c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 14760c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 14776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Emit an intrinsic with a glued value instead of its CC result. 14786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic SDValue emitIntrinsicWithGlue(SelectionDAG &DAG, SDValue Op, 14796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode) { 14806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Copy all operands except the intrinsic ID. 14816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumOps = Op.getNumOperands(); 14826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, 6> Ops; 14836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.reserve(NumOps - 1); 14846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 1; I < NumOps; ++I) 14856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(Op.getOperand(I)); 14866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op->getNumValues() == 1) 14886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, SDLoc(Op), MVT::Glue, Ops); 14896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(Op->getNumValues() == 2 && "Expected exactly one non-CC result"); 14906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDVTList RawVTs = DAG.getVTList(Op->getValueType(0), MVT::Glue); 14916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops); 14926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 14936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// CC is a comparison that will be implemented using an integer or 14951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// floating-point comparison. Return the condition code mask for 14961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// a branch on true. In the integer case, CCMASK_CMP_UO is set for 14971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// unsigned comparisons and clear for signed ones. In the floating-point 14981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 14991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandstatic unsigned CCMaskForCondCode(ISD::CondCode CC) { 15001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#define CONV(X) \ 15011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 15021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 15031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 15041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 15051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (CC) { 15061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 15071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Invalid integer condition!"); 15081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 15091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(EQ); 15101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(NE); 15111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(GT); 15121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(GE); 15131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(LT); 15141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CONV(LE); 15151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 15161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SETO: return SystemZ::CCMASK_CMP_O; 15171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 15181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 15191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#undef CONV 15201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 15211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1522d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford// Return a sequence for getting a 1 from an IPM result when CC has a 1523d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1524d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford// The handling of CC values outside CCValid doesn't matter. 1525d1a4f579bf45aec933c79292b6b9663581438738Richard Sandifordstatic IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1526d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // Deal with cases where the result can be taken directly from a bit 1527d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // of the IPM result. 1528d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1529d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, 0, SystemZ::IPM_CC); 1530d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1531d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1532d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 1533d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // Deal with cases where we can add a value to force the sign bit 1534d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // to contain the right value. Putting the bit in 31 means we can 1535d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // use SRL rather than RISBG(L), and also makes it easier to get a 1536d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // 0/-1 value, so it has priority over the other tests below. 1537d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // 1538d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // These sequences rely on the fact that the upper two bits of the 1539d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // IPM result are zero. 1540d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford uint64_t TopBit = uint64_t(1) << 31; 1541d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1542d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1543d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1544d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1545d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 1546d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_1 1547d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_2))) 1548d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1549d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1550d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1551d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_1 1552d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_2 1553d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_3))) 1554d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1555d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 1556d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // Next try inverting the value and testing a bit. 0/1 could be 1557d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // handled this way too, but we dealt with that case above. 1558d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1559d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(-1, 0, SystemZ::IPM_CC); 1560d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 1561d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // Handle cases where adding a value forces a non-sign bit to contain 1562d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // the right value. 1563d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1564d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1565d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1566d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1567d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 156836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are 1569d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // can be done by inverting the low CC bit and applying one of the 1570d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // sign-based extractions above. 1571d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1572d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1573d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1574d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(1 << SystemZ::IPM_CC, 1575d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford TopBit - (3 << SystemZ::IPM_CC), 31); 1576d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 1577d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_1 1578d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_3))) 1579d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1580d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (CCMask == (CCValid & (SystemZ::CCMASK_0 1581d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_2 1582d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford | SystemZ::CCMASK_3))) 1583d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return IPMConversion(1 << SystemZ::IPM_CC, 1584d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford TopBit - (1 << SystemZ::IPM_CC), 31); 1585d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 1586d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford llvm_unreachable("Unexpected CC combination"); 1587d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford} 1588d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 158936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// If C can be converted to a comparison against zero, adjust the operands 15903237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford// as necessary. 1591de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { 159236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType == SystemZICMP::UnsignedOnly) 15933237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford return; 15943237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford 159536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode()); 15963237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford if (!ConstOp1) 15973237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford return; 15983237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford 15993237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford int64_t Value = ConstOp1->getSExtValue(); 160036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) || 160136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) || 160236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) || 160336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) { 160436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask ^= SystemZ::CCMASK_CMP_EQ; 16056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType()); 16063237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford } 16073237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford} 16083237f88882eed8a67fa679f7071a5441c4306ac3Richard Sandiford 160936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI, 161036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// adjust the operands as necessary. 1611de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, 1612de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Comparison &C) { 16131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // For us to make any changes, it must a comparison between a single-use 16141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // load and a constant. 161536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!C.Op0.hasOneUse() || 161636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0.getOpcode() != ISD::LOAD || 161736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op1.getOpcode() != ISD::Constant) 16181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 16191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 16201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We must have an 8- or 16-bit load. 162136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Load = cast<LoadSDNode>(C.Op0); 16221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 16231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (NumBits != 8 && NumBits != 16) 16241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 16251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 16261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The load must be an extending one and the constant must be within the 16271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // range of the unextended value. 162836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *ConstOp1 = cast<ConstantSDNode>(C.Op1); 162936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t Value = ConstOp1->getZExtValue(); 16301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand uint64_t Mask = (1 << NumBits) - 1; 16311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Load->getExtensionType() == ISD::SEXTLOAD) { 163236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure that ConstOp1 is in range of C.Op0. 163336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t SignedValue = ConstOp1->getSExtValue(); 163436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask) 16351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 163636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::SignedOnly) { 163736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Unsigned comparison between two sign-extended values is equivalent 163836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to unsigned comparison between two zero-extended values. 16391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Value &= Mask; 164036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (NumBits == 8) { 16411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Try to treat the comparison as unsigned, so that we can use CLI. 16421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Adjust CCMask and Value as necessary. 164336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT) 16441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Test whether the high bit of the byte is set. 164536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT; 164636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE) 16471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Test whether the high bit of the byte is clear. 164836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT; 16491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else 16501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // No instruction exists for this combination. 16511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 165236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.ICmpType = SystemZICMP::UnsignedOnly; 16531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 16541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 16551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Value > Mask) 16561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 1657f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // If the constant is in range, we can use any comparison. 1658f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar C.ICmpType = SystemZICMP::Any; 16591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else 16601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return; 16611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 16621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Make sure that the first operand is an i32 of the right extension type. 166336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ? 166436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ISD::SEXTLOAD : 166536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ISD::ZEXTLOAD); 166636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getValueType() != MVT::i32 || 16671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Load->getExtensionType() != ExtType) 166836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 166936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getChain(), Load->getBasePtr(), 167036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getPointerInfo(), Load->getMemoryVT(), 167136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->isVolatile(), Load->isNonTemporal(), 167237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Load->isInvariant(), Load->getAlignment()); 16731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 16741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Make sure that the second operand is an i32 with the right value. 167536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op1.getValueType() != MVT::i32 || 167636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value != ConstOp1->getZExtValue()) 16776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar C.Op1 = DAG.getConstant(Value, DL, MVT::i32); 16781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 16791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 1680aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford// Return true if Op is either an unextended load, or a load suitable 1681aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford// for integer register-memory comparisons of type ICmpType. 1682aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandifordstatic bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 168336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1684aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford if (Load) { 1685aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // There are no instructions to compare a register with a memory byte. 1686aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford if (Load->getMemoryVT() == MVT::i8) 1687aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford return false; 1688aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // Otherwise decide on extension type. 168935c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford switch (Load->getExtensionType()) { 169035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford case ISD::NON_EXTLOAD: 169135c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return true; 169235c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford case ISD::SEXTLOAD: 1693aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford return ICmpType != SystemZICMP::UnsignedOnly; 169435c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford case ISD::ZEXTLOAD: 1695aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford return ICmpType != SystemZICMP::SignedOnly; 169635c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford default: 169735c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford break; 169835c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford } 1699aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford } 170035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return false; 170135c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford} 170235c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 170336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return true if it is better to swap the operands of C. 170436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool shouldSwapCmpOperands(const Comparison &C) { 170535c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // Leave f128 comparisons alone, since they have no memory forms. 170636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getValueType() == MVT::f128) 170735c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return false; 170835c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 170935c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // Always keep a floating-point constant second, since comparisons with 171035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // zero can use LOAD TEST and comparisons with other constants make a 171135c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // natural memory operand. 171236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<ConstantFPSDNode>(C.Op1)) 171335c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return false; 171435c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 171535c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // Never swap comparisons with zero since there are many ways to optimize 171635c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // those later. 171736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 171836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstOp1 && ConstOp1->getZExtValue() == 0) 171936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 172036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 172136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Also keep natural memory operands second if the loaded value is 172236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // only used here. Several comparisons have memory forms. 172336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse()) 172435c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return false; 172535c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 172635c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 172735c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // In that case we generally prefer the memory to be second. 172836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) { 172935c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // The only exceptions are when the second operand is a constant and 173035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford // we can use things like CHHSI. 173136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ConstOp1) 173235c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return true; 1733aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // The unsigned memory-immediate instructions can handle 16-bit 1734aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // unsigned integers. 173536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::SignedOnly && 173636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isUInt<16>(ConstOp1->getZExtValue())) 1737aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford return false; 1738aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // The signed memory-immediate instructions can handle 16-bit 1739aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // signed integers. 174036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::UnsignedOnly && 174136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isInt<16>(ConstOp1->getSExtValue())) 1742aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford return false; 174335c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return true; 174435c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford } 174536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 174636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Try to promote the use of CGFR and CLGFR. 174736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Opcode0 = C.Op0.getOpcode(); 174836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND) 174936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 175036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND) 175136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 175236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.ICmpType != SystemZICMP::SignedOnly && 175336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Opcode0 == ISD::AND && 175436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0.getOperand(1).getOpcode() == ISD::Constant && 175536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff) 175636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 175736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 175835c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford return false; 175935c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford} 176035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 176136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return a version of comparison CC mask CCMask in which the LT and GT 176236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// actions are swapped. 176336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic unsigned reverseCCMask(unsigned CCMask) { 176436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 176536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 176636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 176736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (CCMask & SystemZ::CCMASK_CMP_UO)); 176836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 176936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Check whether C tests for equality between X and Y and whether X - Y 177136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// or Y - X is also computed. In that case it's better to compare the 177236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// result of the subtraction against zero. 1773de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, 1774de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Comparison &C) { 177536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 177636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask == SystemZ::CCMASK_CMP_NE) { 177736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 177836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDNode *N = *I; 177936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (N->getOpcode() == ISD::SUB && 178036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) || 178136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) { 178236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = SDValue(N, 0); 17836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar C.Op1 = DAG.getConstant(0, DL, N->getValueType(0)); 178436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 178536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 178936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 179036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Check whether C compares a floating-point value with zero and if that 179136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// floating-point value is also negated. In this case we can use the 179236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// negation to set CC, so avoiding separate LOAD AND TEST and 179336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// LOAD (NEGATIVE/COMPLEMENT) instructions. 179436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic void adjustForFNeg(Comparison &C) { 179536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1); 179636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C1 && C1->isZero()) { 179736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 179836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDNode *N = *I; 179936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (N->getOpcode() == ISD::FNEG) { 180036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = SDValue(N, 0); 180136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask = reverseCCMask(C.CCMask); 180236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 180336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 180436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 180536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 180636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 180736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 180836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Check whether C compares (shl X, 32) with 0 and whether X is 180936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// also sign-extended. In that case it is better to test the result 181036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// of the sign extension using LTGFR. 181136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 181236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This case is important because InstCombine transforms a comparison 181336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// with (sext (trunc X)) into a comparison with (shl X, 32). 181436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic void adjustForLTGFR(Comparison &C) { 181536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check for a comparison between (shl X, 32) and 0. 181636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getOpcode() == ISD::SHL && 181736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0.getValueType() == MVT::i64 && 181836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op1.getOpcode() == ISD::Constant && 181936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 182036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1)); 182136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C1 && C1->getZExtValue() == 32) { 182236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ShlOp0 = C.Op0.getOperand(0); 182336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // See whether X has any SIGN_EXTEND_INREG uses. 182436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) { 182536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDNode *N = *I; 182636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (N->getOpcode() == ISD::SIGN_EXTEND_INREG && 182736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) { 182836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = SDValue(N, 0); 182936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 183036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 183136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 183236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 183336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 183436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 183536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// If C compares the truncation of an extending load, try to compare 183736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// the untruncated value instead. This exposes more opportunities to 183836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// reuse CC. 1839de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, 1840de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Comparison &C) { 184136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getOpcode() == ISD::TRUNCATE && 184236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0.getOperand(0).getOpcode() == ISD::LOAD && 184336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op1.getOpcode() == ISD::Constant && 184436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 184536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *L = cast<LoadSDNode>(C.Op0.getOperand(0)); 184636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (L->getMemoryVT().getStoreSizeInBits() 184736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines <= C.Op0.getValueType().getSizeInBits()) { 184836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Type = L->getExtensionType(); 184936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) || 185036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) { 185136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = C.Op0.getOperand(0); 18526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType()); 185336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 185436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 185536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 185636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 185736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1858856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// Return true if shift operation N has an in-range constant shift value. 1859856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// Store it in ShiftVal if so. 1860856bf594338567a592086fe782f2f51650e4e294Richard Sandifordstatic bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 186136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1862856bf594338567a592086fe782f2f51650e4e294Richard Sandiford if (!Shift) 1863856bf594338567a592086fe782f2f51650e4e294Richard Sandiford return false; 1864856bf594338567a592086fe782f2f51650e4e294Richard Sandiford 1865856bf594338567a592086fe782f2f51650e4e294Richard Sandiford uint64_t Amount = Shift->getZExtValue(); 1866856bf594338567a592086fe782f2f51650e4e294Richard Sandiford if (Amount >= N.getValueType().getSizeInBits()) 1867856bf594338567a592086fe782f2f51650e4e294Richard Sandiford return false; 1868856bf594338567a592086fe782f2f51650e4e294Richard Sandiford 1869856bf594338567a592086fe782f2f51650e4e294Richard Sandiford ShiftVal = Amount; 1870856bf594338567a592086fe782f2f51650e4e294Richard Sandiford return true; 1871856bf594338567a592086fe782f2f51650e4e294Richard Sandiford} 1872856bf594338567a592086fe782f2f51650e4e294Richard Sandiford 1873856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// Check whether an AND with Mask is suitable for a TEST UNDER MASK 1874856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// instruction and whether the CC value is descriptive enough to handle 1875856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// a comparison of type Opcode between the AND result and CmpVal. 1876856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// CCMask says which comparison result is being tested and BitSize is 1877856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// the number of bits in the operands. If TEST UNDER MASK can be used, 1878856bf594338567a592086fe782f2f51650e4e294Richard Sandiford// return the corresponding CC mask, otherwise return 0. 1879aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandifordstatic unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 1880aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford uint64_t Mask, uint64_t CmpVal, 1881aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford unsigned ICmpType) { 18828bce43648be1156fdced590beb81aed3915762f1Richard Sandiford assert(Mask != 0 && "ANDs with zero should have been removed by now"); 18838bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 1884856bf594338567a592086fe782f2f51650e4e294Richard Sandiford // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 1885856bf594338567a592086fe782f2f51650e4e294Richard Sandiford if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 1886856bf594338567a592086fe782f2f51650e4e294Richard Sandiford !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 1887856bf594338567a592086fe782f2f51650e4e294Richard Sandiford return 0; 1888856bf594338567a592086fe782f2f51650e4e294Richard Sandiford 18898bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Work out the masks for the lowest and highest bits. 18908bce43648be1156fdced590beb81aed3915762f1Richard Sandiford unsigned HighShift = 63 - countLeadingZeros(Mask); 18918bce43648be1156fdced590beb81aed3915762f1Richard Sandiford uint64_t High = uint64_t(1) << HighShift; 18928bce43648be1156fdced590beb81aed3915762f1Richard Sandiford uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 18938bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 18948bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Signed ordered comparisons are effectively unsigned if the sign 18958bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // bit is dropped. 1896aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 18978bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 18988bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Check for equality comparisons with 0, or the equivalent. 18998bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CmpVal == 0) { 19008bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_EQ) 19018bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_0; 19028bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_NE) 19038bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_1; 19048bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 1905de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) { 19068bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LT) 19078bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_0; 19088bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GE) 19098bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_1; 19108bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19118bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (EffectivelyUnsigned && CmpVal < Low) { 19128bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LE) 19138bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_0; 19148bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GT) 19158bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_1; 19168bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19178bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 19188bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Check for equality comparisons with the mask, or the equivalent. 19198bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CmpVal == Mask) { 19208bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_EQ) 19218bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_1; 19228bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_NE) 19238bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_0; 19248bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19258bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 19268bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GT) 19278bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_1; 19288bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LE) 19298bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_0; 19308bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19318bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 19328bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GE) 19338bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_ALL_1; 19348bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LT) 19358bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_SOME_0; 19368bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19378bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 19388bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Check for ordered comparisons with the top bit. 19398bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 19408bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LE) 19418bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MSB_0; 19428bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GT) 19438bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MSB_1; 19448bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19458bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 19468bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_LT) 19478bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MSB_0; 19488bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_GE) 19498bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MSB_1; 19508bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19518bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 19528bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // If there are just two bits, we can do equality checks for Low and High 19538bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // as well. 19548bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (Mask == Low + High) { 19558bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 19568bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MIXED_MSB_0; 19578bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 19588bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 19598bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 19608bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MIXED_MSB_1; 19618bce43648be1156fdced590beb81aed3915762f1Richard Sandiford if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 19628bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 19638bce43648be1156fdced590beb81aed3915762f1Richard Sandiford } 19648bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 19658bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Looks like we've exhausted our options. 19668bce43648be1156fdced590beb81aed3915762f1Richard Sandiford return 0; 19678bce43648be1156fdced590beb81aed3915762f1Richard Sandiford} 19688bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 196936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// See whether C can be implemented as a TEST UNDER MASK instruction. 197036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Update the arguments with the TM version if so. 1971de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, 1972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Comparison &C) { 19738bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Check that we have a comparison with a constant. 197436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 197536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ConstOp1) 1976477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford return; 197736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t CmpVal = ConstOp1->getZExtValue(); 1978477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford 1979477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford // Check whether the nonconstant input is an AND with a constant mask. 198036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Comparison NewC(C); 198136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t MaskVal; 1982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Mask = nullptr; 198336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getOpcode() == ISD::AND) { 198436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op0 = C.Op0.getOperand(0); 198536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op1 = C.Op0.getOperand(1); 198636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Mask = dyn_cast<ConstantSDNode>(NewC.Op1); 198736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Mask) 198836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 198936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskVal = Mask->getZExtValue(); 199036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 199136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // There is no instruction to compare with a 64-bit immediate 199236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // so use TMHH instead if possible. We need an unsigned ordered 199336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // comparison with an i64 immediate. 199436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NewC.Op0.getValueType() != MVT::i64 || 199536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.CCMask == SystemZ::CCMASK_CMP_EQ || 199636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.CCMask == SystemZ::CCMASK_CMP_NE || 199736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.ICmpType == SystemZICMP::SignedOnly) 199836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 199936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Convert LE and GT comparisons into LT and GE. 200036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NewC.CCMask == SystemZ::CCMASK_CMP_LE || 200136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.CCMask == SystemZ::CCMASK_CMP_GT) { 200236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CmpVal == uint64_t(-1)) 200336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 200436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpVal += 1; 200536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ; 200636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 200736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the low N bits of Op1 are zero than the low N bits of Op0 can 200836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // be masked off without changing the result. 200936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskVal = -(CmpVal & -CmpVal); 201036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.ICmpType = SystemZICMP::UnsignedOnly; 201136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 20120c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (!MaskVal) 20130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return; 2014477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford 20158bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // Check whether the combination of mask, comparison value and comparison 20168bce43648be1156fdced590beb81aed3915762f1Richard Sandiford // type are suitable. 201736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned BitSize = NewC.Op0.getValueType().getSizeInBits(); 2018856bf594338567a592086fe782f2f51650e4e294Richard Sandiford unsigned NewCCMask, ShiftVal; 201936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NewC.ICmpType != SystemZICMP::SignedOnly && 202036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op0.getOpcode() == ISD::SHL && 202136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isSimpleShift(NewC.Op0, ShiftVal) && 202236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 202336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskVal >> ShiftVal, 2024856bf594338567a592086fe782f2f51650e4e294Richard Sandiford CmpVal >> ShiftVal, 2025856bf594338567a592086fe782f2f51650e4e294Richard Sandiford SystemZICMP::Any))) { 202636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op0 = NewC.Op0.getOperand(0); 202736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskVal >>= ShiftVal; 202836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (NewC.ICmpType != SystemZICMP::SignedOnly && 202936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op0.getOpcode() == ISD::SRL && 203036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isSimpleShift(NewC.Op0, ShiftVal) && 203136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 2032856bf594338567a592086fe782f2f51650e4e294Richard Sandiford MaskVal << ShiftVal, 2033856bf594338567a592086fe782f2f51650e4e294Richard Sandiford CmpVal << ShiftVal, 2034856bf594338567a592086fe782f2f51650e4e294Richard Sandiford SystemZICMP::UnsignedOnly))) { 203536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.Op0 = NewC.Op0.getOperand(0); 203636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskVal <<= ShiftVal; 2037856bf594338567a592086fe782f2f51650e4e294Richard Sandiford } else { 203836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal, 203936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewC.ICmpType); 2040856bf594338567a592086fe782f2f51650e4e294Richard Sandiford if (!NewCCMask) 2041856bf594338567a592086fe782f2f51650e4e294Richard Sandiford return; 2042856bf594338567a592086fe782f2f51650e4e294Richard Sandiford } 20438bce43648be1156fdced590beb81aed3915762f1Richard Sandiford 2044477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford // Go ahead and make the change. 204536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Opcode = SystemZISD::TM; 204636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op0 = NewC.Op0; 204736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Mask && Mask->getZExtValue() == MaskVal) 204836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op1 = SDValue(Mask, 0); 204936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 20506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType()); 205136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCValid = SystemZ::CCMASK_TM; 205236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask = NewCCMask; 205336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 205436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 20550c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Return a Comparison that tests the condition-code result of intrinsic 20560c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// node Call against constant integer CC using comparison code Cond. 20570c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Opcode is the opcode of the SystemZISD operation for the intrinsic 20580c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// and CCValid is the set of possible condition-code results. 20590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarstatic Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode, 20600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Call, unsigned CCValid, uint64_t CC, 20610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar ISD::CondCode Cond) { 20620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Comparison C(Call, SDValue()); 20630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar C.Opcode = Opcode; 20640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar C.CCValid = CCValid; 20650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Cond == ISD::SETEQ) 20660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3. 20670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar C.CCMask = CC < 4 ? 1 << (3 - CC) : 0; 20680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (Cond == ISD::SETNE) 20690c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // ...and the inverse of that. 20700c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1; 20710c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (Cond == ISD::SETLT || Cond == ISD::SETULT) 20720c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3, 20730c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // always true for CC>3. 2074f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1; 20750c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (Cond == ISD::SETGE || Cond == ISD::SETUGE) 20760c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // ...and the inverse of that. 2077f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0; 20780c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (Cond == ISD::SETLE || Cond == ISD::SETULE) 20790c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true), 20800c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // always true for CC>3. 2081f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1; 20820c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (Cond == ISD::SETGT || Cond == ISD::SETUGT) 20830c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // ...and the inverse of that. 2084f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0; 20850c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else 20860c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar llvm_unreachable("Unexpected integer comparison type"); 20870c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar C.CCMask &= CCValid; 20880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return C; 20890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 20900c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 209136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1. 209236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 2093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ISD::CondCode Cond, const SDLoc &DL) { 20940c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (CmpOp1.getOpcode() == ISD::Constant) { 20950c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue(); 20960c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Opcode, CCValid; 20970c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN && 20980c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) && 20990c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid)) 21000c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond); 21016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 21026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 && 21036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar isIntrinsicWithCC(CmpOp0, Opcode, CCValid)) 21046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond); 21050c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 210636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Comparison C(CmpOp0, CmpOp1); 210736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask = CCMaskForCondCode(Cond); 210836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Op0.getValueType().isFloatingPoint()) { 210936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCValid = SystemZ::CCMASK_FCMP; 211036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Opcode = SystemZISD::FCMP; 211136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines adjustForFNeg(C); 2112aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford } else { 211336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCValid = SystemZ::CCMASK_ICMP; 211436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Opcode = SystemZISD::ICMP; 2115aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // Choose the type of comparison. Equality and inequality tests can 2116aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // use either signed or unsigned comparisons. The choice also doesn't 2117aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // matter if both sign bits are known to be clear. In those cases we 2118aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // want to give the main isel code the freedom to choose whichever 2119aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford // form fits best. 212036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 212136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask == SystemZ::CCMASK_CMP_NE || 212236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1))) 212336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.ICmpType = SystemZICMP::Any; 212436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (C.CCMask & SystemZ::CCMASK_CMP_UO) 212536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.ICmpType = SystemZICMP::UnsignedOnly; 2126aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford else 212736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.ICmpType = SystemZICMP::SignedOnly; 212836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask &= ~SystemZ::CCMASK_CMP_UO; 21296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar adjustZeroCmp(DAG, DL, C); 21306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar adjustSubwordCmp(DAG, DL, C); 21316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar adjustForSubtraction(DAG, DL, C); 213236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines adjustForLTGFR(C); 21336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar adjustICmpTruncate(DAG, DL, C); 21341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 21351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 213636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (shouldSwapCmpOperands(C)) { 213736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::swap(C.Op0, C.Op1); 213836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask = reverseCCMask(C.CCMask); 213935c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford } 214035c93e4e42d7a35a90e89211fa62f478e25ba0a4Richard Sandiford 21416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar adjustForTestUnderMask(DAG, DL, C); 214236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return C; 214336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 214436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Emit the comparison instruction described by C. 2146de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { 21470c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (!C.Op1.getNode()) { 21480c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Op; 21490c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar switch (C.Op0.getOpcode()) { 21500c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case ISD::INTRINSIC_W_CHAIN: 21510c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op = emitIntrinsicWithChainAndGlue(DAG, C.Op0, C.Opcode); 21520c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar break; 21536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::INTRINSIC_WO_CHAIN: 21546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = emitIntrinsicWithGlue(DAG, C.Op0, C.Opcode); 21556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 21560c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar default: 21570c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar llvm_unreachable("Invalid comparison operands"); 21580c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 21590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return SDValue(Op.getNode(), Op->getNumValues() - 1); 21600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 216136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Opcode == SystemZISD::ICMP) 216236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(SystemZISD::ICMP, DL, MVT::Glue, C.Op0, C.Op1, 21636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(C.ICmpType, DL, MVT::i32)); 216436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Opcode == SystemZISD::TM) { 216536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 216636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 216736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(SystemZISD::TM, DL, MVT::Glue, C.Op0, C.Op1, 21686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(RegisterOnly, DL, MVT::i32)); 216936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(C.Opcode, DL, MVT::Glue, C.Op0, C.Op1); 21711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 21721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2173df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford// Implement a 32-bit *MUL_LOHI operation by extending both operands to 2174df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford// 64 bits. Extend is the extension type to use. Store the high part 2175df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford// in Hi and the low part in Lo. 2176de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, 2177de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op0, SDValue Op1, SDValue &Hi, 2178de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue &Lo) { 2179df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 2180df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 2181df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 21826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, 21836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, MVT::i64)); 2184df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 2185df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 2186df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford} 2187df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford 21881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Lower a binary operation that produces two VT results, one in each 21891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 21901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 21911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// on the extended Op0 and (unextended) Op1. Store the even register result 21921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// in Even and the odd register result in Odd. 2193de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 2194de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Extend, unsigned Opcode, SDValue Op0, 2195de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op1, SDValue &Even, SDValue &Odd) { 21961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 21971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 21981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue(In128, 0), Op1); 21991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand bool Is32Bit = is32Bit(VT); 22007c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 22017c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 22021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 22031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 220436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return an i32 value that is 1 if the CC value produced by Glue is 220536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// in the mask CCMask and 0 otherwise. CC is known to have a value 220636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// in CCValid, so other values can be ignored. 2207de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue Glue, 220836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CCValid, unsigned CCMask) { 2209d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford IPMConversion Conversion = getIPMConversion(CCValid, CCMask); 2210d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford SDValue Result = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue); 2211d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 2212d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (Conversion.XORValue) 2213d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford Result = DAG.getNode(ISD::XOR, DL, MVT::i32, Result, 22146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Conversion.XORValue, DL, MVT::i32)); 2215d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 2216d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (Conversion.AddValue) 2217d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford Result = DAG.getNode(ISD::ADD, DL, MVT::i32, Result, 22186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Conversion.AddValue, DL, MVT::i32)); 2219d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 2220d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford // The SHR/AND sequence should get optimized to an RISBG. 2221d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford Result = DAG.getNode(ISD::SRL, DL, MVT::i32, Result, 22226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Conversion.Bit, DL, MVT::i32)); 2223d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford if (Conversion.Bit != 31) 2224d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, 22256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32)); 2226d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return Result; 2227d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford} 2228d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford 22296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return the SystemISD vector comparison operation for CC, or 0 if it cannot 22306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// be done directly. IsFP is true if CC is for a floating-point rather than 22316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// integer comparison. 22326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) { 22336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch (CC) { 22346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETOEQ: 22356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETEQ: 22366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE; 22376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETOGE: 22396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETGE: 22406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0); 22416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETOGT: 22436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETGT: 22446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH; 22456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETUGT: 22476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL; 22486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar default: 22506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return 0; 22516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 22526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 22536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return the SystemZISD vector comparison operation for CC or its inverse, 22556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// or 0 if neither can be done directly. Indicate in Invert whether the 22566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// result is for the inverse of CC. IsFP is true if CC is for a 22576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// floating-point rather than integer comparison. 22586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP, 22596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool &Invert) { 22606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (unsigned Opcode = getVectorComparison(CC, IsFP)) { 22616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Invert = false; 22626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Opcode; 22636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 22646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CC = ISD::getSetCCInverse(CC, !IsFP); 22666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (unsigned Opcode = getVectorComparison(CC, IsFP)) { 22676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Invert = true; 22686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Opcode; 22696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 22706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return 0; 22726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 22736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return a v2f64 that contains the extended form of elements Start and Start+1 22756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// of v4f32 value Op. 2276de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, 22776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op) { 22786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Mask[] = { Start, -1, Start + 1, -1 }; 22796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask); 22806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op); 22816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 22826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 22836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode, 22846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// producing a result of type VT. 2285de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &DL, 22866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT, SDValue CmpOp0, SDValue CmpOp1) { 22876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // There is no hardware support for v4f32, so extend the vector into 22886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // two v2f64s and compare those. 22896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (CmpOp0.getValueType() == MVT::v4f32) { 22906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0); 22916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0); 22926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1); 22936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1); 22946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1); 22956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1); 22966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes); 22976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 22986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1); 22996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 23006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 23016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing 23026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// an integer mask of type VT. 2303de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 23046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar ISD::CondCode CC, SDValue CmpOp0, 23056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CmpOp1) { 23066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool IsFP = CmpOp0.getValueType().isFloatingPoint(); 23076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool Invert = false; 23086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Cmp; 23096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch (CC) { 23106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle tests for order using (or (ogt y x) (oge x y)). 23116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETUO: 23126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Invert = true; 23136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETO: { 23146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(IsFP && "Unexpected integer comparison"); 23156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); 23166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1); 23176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE); 23186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 23196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 23206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 23216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle <> tests using (or (ogt y x) (ogt x y)). 23226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETUEQ: 23236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Invert = true; 23246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SETONE: { 23256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(IsFP && "Unexpected integer comparison"); 23266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); 23276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1); 23286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT); 23296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 23306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 23316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 23326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise a single comparison is enough. It doesn't really 23336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // matter whether we try the inversion or the swap first, since 23346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // there are no cases where both work. 23356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar default: 23366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert)) 23376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1); 23386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else { 23396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CC = ISD::getSetCCSwappedOperands(CC); 23406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert)) 23416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0); 23426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 23436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar llvm_unreachable("Unhandled comparison"); 23446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 23456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 23466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 23476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Invert) { 23486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Mask = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8, 23496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(65535, DL, MVT::i32)); 23506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Mask = DAG.getNode(ISD::BITCAST, DL, VT, Mask); 23516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask); 23526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 23536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Cmp; 23546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 23556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 235636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue SystemZTargetLowering::lowerSETCC(SDValue Op, 235736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 235836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue CmpOp0 = Op.getOperand(0); 235936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue CmpOp1 = Op.getOperand(1); 236036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 236136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 23626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 23636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT.isVector()) 23646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1); 236536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 236736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Glue = emitCmp(DAG, DL, C); 236836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 236936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 237036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 23721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 23731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue CmpOp0 = Op.getOperand(2); 23741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue CmpOp1 = Op.getOperand(3); 23751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Dest = Op.getOperand(4); 2376ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 23771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 23786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 237936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Glue = emitCmp(DAG, DL, C); 23801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 23816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32), 23826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, Glue); 238336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 238436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return true if Pos is CmpOp and Neg is the negative of CmpOp, 238636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// allowing Pos and Neg to be wider than CmpOp. 238736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) { 238836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return (Neg.getOpcode() == ISD::SUB && 238936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Neg.getOperand(0).getOpcode() == ISD::Constant && 239036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 && 239136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Neg.getOperand(1) == Pos && 239236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Pos == CmpOp || 239336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Pos.getOpcode() == ISD::SIGN_EXTEND && 239436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Pos.getOperand(0) == CmpOp))); 239536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 239636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 239736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return the absolute or negative absolute of Op; IsNegative decides which. 2398de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, 239936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsNegative) { 240036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op); 240136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IsNegative) 240236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(), 24036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, Op.getValueType()), Op); 240436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Op; 24051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 24061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 24081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 24091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue CmpOp0 = Op.getOperand(0); 24101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue CmpOp1 = Op.getOperand(1); 24111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue TrueOp = Op.getOperand(2); 24121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue FalseOp = Op.getOperand(3); 24131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 2414ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 24151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 241736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 241836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check for absolute and negative-absolute selections, including those 241936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // where the comparison value is sign-extended (for LPGFR and LNGFR). 242036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This check supplements the one in DAGCombiner. 242136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (C.Opcode == SystemZISD::ICMP && 242236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask != SystemZ::CCMASK_CMP_EQ && 242336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask != SystemZ::CCMASK_CMP_NE && 242436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.Op1.getOpcode() == ISD::Constant && 242536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 242636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isAbsolute(C.Op0, TrueOp, FalseOp)) 242736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT); 242836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isAbsolute(C.Op0, FalseOp, TrueOp)) 242936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT); 243036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 243136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 243236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Glue = emitCmp(DAG, DL, C); 243336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 243436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Special case for handling -1/0 results. The shifts we use here 243536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // should get optimized with the IPM conversion sequence. 243636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *TrueC = dyn_cast<ConstantSDNode>(TrueOp); 243736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *FalseC = dyn_cast<ConstantSDNode>(FalseOp); 243836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TrueC && FalseC) { 243936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t TrueVal = TrueC->getSExtValue(); 244036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t FalseVal = FalseC->getSExtValue(); 244136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((TrueVal == -1 && FalseVal == 0) || (TrueVal == 0 && FalseVal == -1)) { 244236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Invert the condition if we want -1 on false. 244336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TrueVal == 0) 244436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines C.CCMask ^= C.CCValid; 244536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Result = emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 244636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = Op.getValueType(); 244736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Extend the result to VT. Upper bits are ignored. 244836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!is32Bit(VT)) 244936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Result); 245036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Sign-extend from the low bit. 24516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue ShAmt = DAG.getConstant(VT.getSizeInBits() - 1, DL, MVT::i32); 245236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Result, ShAmt); 245336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(ISD::SRA, DL, VT, Shl, ShAmt); 245436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 245536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 24561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32), 24586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(C.CCMask, DL, MVT::i32), Glue}; 24591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 2461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, Ops); 24621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 24631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 24651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 2466ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Node); 24671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const GlobalValue *GV = Node->getGlobal(); 24681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t Offset = Node->getOffset(); 2469f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2470c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines CodeModel::Model CM = DAG.getTarget().getCodeModel(); 24711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result; 2473de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Subtarget.isPC32DBLSymbol(GV, CM)) { 24748dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford // Assign anchors at 1<<12 byte boundaries. 24758dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford uint64_t Anchor = Offset & ~uint64_t(0xfff); 24768dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 24778dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 24788dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford 24798dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford // The offset can be folded into the address if it is aligned to a halfword. 24808dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford Offset -= Anchor; 24818dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford if (Offset != 0 && (Offset & 1) == 0) { 24828dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 24838dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 24841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Offset = 0; 24851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 24861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else { 24871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 24881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 24891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 2490f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2491f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, false, 0); 24921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 24931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 24941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // If there was a non-zero offset that we didn't fold, create an explicit 24951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // addition for it. 24961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Offset != 0) 24971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 24986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Offset, DL, PtrVT)); 24991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 25001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Result; 25011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 25021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2503ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesSDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node, 2504ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SelectionDAG &DAG, 2505ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned Opcode, 2506ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue GOTOffset) const { 2507ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDLoc DL(Node); 2508f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2509ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue Chain = DAG.getEntryNode(); 2510ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue Glue; 2511ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2512ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12. 2513ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT); 2514ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue); 2515ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Glue = Chain.getValue(1); 2516ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue); 2517ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Glue = Chain.getValue(1); 2518ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2519ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The first call operand is the chain and the second is the TLS symbol. 2520ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<SDValue, 8> Ops; 2521ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(Chain); 2522ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL, 2523ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Node->getValueType(0), 2524ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 0, 0)); 2525ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2526ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Add argument registers to the end of the list so that they are 2527ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // known live into the call. 2528ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT)); 2529ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT)); 2530ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2531ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Add a register mask operand representing the call-preserved registers. 2532ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 25334c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const uint32_t *Mask = 25344c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C); 2535ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(Mask && "Missing call preserved mask for calling convention"); 2536ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(DAG.getRegisterMask(Mask)); 2537ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2538ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Glue the call to the argument copies. 2539ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Ops.push_back(Glue); 2540ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2541ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Emit the call. 2542ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2543ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Chain = DAG.getNode(Opcode, DL, NodeTys, Ops); 2544ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Glue = Chain.getValue(1); 2545ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2546ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Copy the return value from %r2. 2547ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue); 2548ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2549ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2550de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL, 2551de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) const { 2552f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 25531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 25541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The high part of the thread pointer is in access register 0. 25551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 25566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)); 25571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 25581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 25591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The low part of the thread pointer is in access register 1. 25601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 25616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32)); 25621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 25631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 25641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Merge them into a single 64-bit address. 25651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 25666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, PtrVT)); 2567de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 2568de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 2569de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2570de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 2571de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) const { 2572de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (DAG.getTarget().Options.EmulatedTLS) 2573de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return LowerToTLSEmulatedModel(Node, DAG); 2574de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Node); 2575de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const GlobalValue *GV = Node->getGlobal(); 2576de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2577de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLSModel::Model model = DAG.getTarget().getTLSModel(GV); 2578de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue TP = lowerThreadPointer(DL, DAG); 25801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2581ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Get the offset of GA from the thread pointer, based on the TLS model. 2582ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue Offset; 2583ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines switch (model) { 2584ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case TLSModel::GeneralDynamic: { 2585ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Load the GOT offset of the tls_index (module ID / per-symbol offset). 2586ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue *CPV = 2587ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue::Create(GV, SystemZCP::TLSGD); 2588ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2589ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2590f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Offset = DAG.getLoad( 2591f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PtrVT, DL, DAG.getEntryNode(), Offset, 2592f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false, 2593f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 2594ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2595ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Call __tls_get_offset to retrieve the offset. 2596ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset); 2597ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 2598ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2599ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2600ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case TLSModel::LocalDynamic: { 2601ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Load the GOT offset of the module ID. 2602ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue *CPV = 2603ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue::Create(GV, SystemZCP::TLSLDM); 2604ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2605ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2606f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Offset = DAG.getLoad( 2607f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PtrVT, DL, DAG.getEntryNode(), Offset, 2608f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false, 2609f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 2610ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Call __tls_get_offset to retrieve the module base offset. 2612ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset); 2613ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Note: The SystemZLDCleanupPass will remove redundant computations 2615ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // of the module base offset. Count total number of local-dynamic 2616ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // accesses to trigger execution of that pass. 2617ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZMachineFunctionInfo* MFI = 2618ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>(); 2619ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MFI->incNumLocalDynamicTLSAccesses(); 2620ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2621ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Add the per-symbol offset. 2622ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CPV = SystemZConstantPoolValue::Create(GV, SystemZCP::DTPOFF); 2623ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2624ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, 8); 2625f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DTPOffset = DAG.getLoad( 2626f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PtrVT, DL, DAG.getEntryNode(), DTPOffset, 2627f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false, 2628f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 26291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2630ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset); 2631ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 2632ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2633ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2634ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case TLSModel::InitialExec: { 2635ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Load the offset from the GOT. 2636ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 2637ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZII::MO_INDNTPOFF); 2638ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset); 2639f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset, 2640f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getGOT(DAG.getMachineFunction()), 2641ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines false, false, false, 0); 2642ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 2643ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2644ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2645ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case TLSModel::LocalExec: { 2646ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Force the offset into the constant pool and load it from there. 2647ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue *CPV = 2648ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 2649ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 2650ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2651f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Offset = DAG.getLoad( 2652f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PtrVT, DL, DAG.getEntryNode(), Offset, 2653f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false, 2654f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 2655ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 2656ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2657ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 26581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Add the base and offset together. 26601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 26611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 26621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 26641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 2665ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Node); 26661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const BlockAddress *BA = Node->getBlockAddress(); 26671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t Offset = Node->getOffset(); 2668f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 26691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 26711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 26721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Result; 26731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 26741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 26761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 2677ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(JT); 2678f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 26791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 26801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Use LARL to load the address of the table. 26821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 26831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 26841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 26861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 2687ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(CP); 2688f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 26891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result; 26911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (CP->isMachineConstantPoolEntry()) 26921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 2693f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CP->getAlignment()); 26941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else 26951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 2696f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CP->getAlignment(), CP->getOffset()); 26971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 26981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Use LARL to load the address of the constant pool entry. 26991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 27001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 27011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2702de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op, 2703de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) const { 2704de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFunction &MF = DAG.getMachineFunction(); 2705de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFrameInfo *MFI = MF.getFrameInfo(); 2706de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MFI->setFrameAddressIsTaken(true); 2707de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2708de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Op); 2709de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 2710de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2711de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2712de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the back chain frame index has not been allocated yet, do so. 2713de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SystemZMachineFunctionInfo *FI = MF.getInfo<SystemZMachineFunctionInfo>(); 2714de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int BackChainIdx = FI->getFramePointerSaveIndex(); 2715de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!BackChainIdx) { 2716de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // By definition, the frame address is the address of the back chain. 2717de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BackChainIdx = MFI->CreateFixedObject(8, -SystemZMC::CallFrameSize, false); 2718de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar FI->setFramePointerSaveIndex(BackChainIdx); 2719de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 2720de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT); 2721de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2722de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // FIXME The frontend should detect this case. 2723de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Depth > 0) { 2724de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar report_fatal_error("Unsupported stack frame traversal count"); 2725de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 2726de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2727de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return BackChain; 2728de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 2729de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2730de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op, 2731de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) const { 2732de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFunction &MF = DAG.getMachineFunction(); 2733de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFrameInfo *MFI = MF.getFrameInfo(); 2734de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MFI->setReturnAddressIsTaken(true); 2735de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2736de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 2737de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 2738de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2739de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Op); 2740de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 2741de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2742de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2743de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // FIXME The frontend should detect this case. 2744de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Depth > 0) { 2745de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar report_fatal_error("Unsupported stack frame traversal count"); 2746de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 2747de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2748de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Return R14D, which has the return address. Mark it an implicit live-in. 2749de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass); 2750de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT); 2751de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 2752de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 27531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 27541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 2755ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 27561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue In = Op.getOperand(0); 27571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT InVT = In.getValueType(); 27581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT ResVT = Op.getValueType(); 27591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 27606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Convert loads directly. This is normally done by DAGCombiner, 27616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // but we need this case for bitcasts that are created during lowering 27626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // and which are then lowered themselves. 27636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (auto *LoadN = dyn_cast<LoadSDNode>(In)) 27646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getLoad(ResVT, DL, LoadN->getChain(), LoadN->getBasePtr(), 27656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar LoadN->getMemOperand()); 27666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 27671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (InVT == MVT::i32 && ResVT == MVT::f32) { 27685fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford SDValue In64; 27695fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford if (Subtarget.hasHighWord()) { 27705fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, 27715fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford MVT::i64); 27725fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 27735fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford MVT::i64, SDValue(U64, 0), In); 27745fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford } else { 27755fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 27765fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, 27776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, MVT::i64)); 27785fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford } 27795fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); 27806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getTargetExtractSubreg(SystemZ::subreg_r32, 27817c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford DL, MVT::f32, Out64); 27821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 27831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (InVT == MVT::f32 && ResVT == MVT::i32) { 27841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 27856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_r32, DL, 27867c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford MVT::f64, SDValue(U64, 0), In); 27877c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 27885fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford if (Subtarget.hasHighWord()) 27895fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, 27905fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford MVT::i32, Out64); 27915fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, 27926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, MVT::i64)); 27935fb8d3144fa5da2b8392b56136163809d8df0527Richard Sandiford return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 27941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 27951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unexpected bitcast combination"); 27961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 27971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 27981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 27991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 28001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 28011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZMachineFunctionInfo *FuncInfo = 28021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MF.getInfo<SystemZMachineFunctionInfo>(); 2803f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 28041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Chain = Op.getOperand(0); 28061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Addr = Op.getOperand(1); 28071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 2808ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 28091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The initial values of each field. 28111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const unsigned NumFields = 4; 28121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Fields[NumFields] = { 28136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT), 28146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT), 28151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 28161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 28171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand }; 28181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Store each field into its respective slot. 28201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue MemOps[NumFields]; 28211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Offset = 0; 28221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand for (unsigned I = 0; I < NumFields; ++I) { 28231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue FieldAddr = Addr; 28241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Offset != 0) 28251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 28266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(Offset, DL)); 28271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 28281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachinePointerInfo(SV, Offset), 28291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand false, false, 0); 28301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Offset += 8; 28311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 2832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 28331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 28341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 28361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 28371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Chain = Op.getOperand(0); 28381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue DstPtr = Op.getOperand(1); 28391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue SrcPtr = Op.getOperand(2); 28401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 28411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 2842ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 28431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL), 28451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 28460c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar /*isTailCall*/false, 28471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 28481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 28491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering:: 28511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandlowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 2852f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); 2853de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFunction &MF = DAG.getMachineFunction(); 2854de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool RealignOpt = !MF.getFunction()-> hasFnAttribute("no-realign-stack"); 2855de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool StoreBackchain = MF.getFunction()->hasFnAttribute("backchain"); 2856f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 28571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Chain = Op.getOperand(0); 28581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Size = Op.getOperand(1); 2859f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Align = Op.getOperand(2); 2860ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 28611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2862f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // If user has set the no alignment function attribute, ignore 2863f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // alloca alignments. 2864f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t AlignVal = (RealignOpt ? 2865f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar dyn_cast<ConstantSDNode>(Align)->getZExtValue() : 0); 2866f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 2867f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t StackAlign = TFI->getStackAlignment(); 2868f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t RequiredAlign = std::max(AlignVal, StackAlign); 2869f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t ExtraAlignSpace = RequiredAlign - StackAlign; 2870f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 28711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned SPReg = getStackPointerRegisterToSaveRestore(); 2872f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue NeededSpace = Size; 28731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get a reference to the stack pointer. 28751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 28761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2877de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If we need a backchain, save it now. 2878de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Backchain; 2879de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (StoreBackchain) 2880de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo(), 2881de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, false, 0); 2882de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2883f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Add extra space for alignment if needed. 2884f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (ExtraAlignSpace) 2885f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace, 2886de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); 2887f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 28881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the new stack pointer value. 2889f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace); 28901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Copy the new stack pointer back. 28921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 28931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 28941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // The allocated data lives above the 160 bytes allocated for the standard 28951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // frame, plus any outgoing stack arguments. We don't know how much that 28961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // amounts to yet, so emit a special ADJDYNALLOC placeholder. 28971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 28981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 28991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2900f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Dynamically realign if needed. 2901f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (RequiredAlign > StackAlign) { 2902f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Result = 2903f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, MVT::i64, Result, 2904f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); 2905f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Result = 2906f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::AND, DL, MVT::i64, Result, 2907f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64)); 2908f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 2909f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 2910de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (StoreBackchain) 2911de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo(), 2912de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, 0); 2913de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 29141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[2] = { Result, Chain }; 2915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 29161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 29171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2918de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET( 2919de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op, SelectionDAG &DAG) const { 2920de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Op); 2921de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2922de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 2923de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 2924de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2925df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard SandifordSDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 29261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 29271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT VT = Op.getValueType(); 2928ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 2929df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue Ops[2]; 2930df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford if (is32Bit(VT)) 2931df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // Just do a normal 64-bit multiplication and extract the results. 2932df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // We define this so that it can be used for constant division. 2933df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 2934df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Op.getOperand(1), Ops[1], Ops[0]); 2935df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford else { 2936df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // Do a full 128-bit multiplication based on UMUL_LOHI64: 2937df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // 2938df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 2939df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // 2940df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // but using the fact that the upper halves are either all zeros 2941df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // or all ones: 2942df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // 2943df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 2944df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // 2945df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // and grouping the right terms together since they are quicker than the 2946df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // multiplication: 2947df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // 2948df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 29496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue C63 = DAG.getConstant(63, DL, MVT::i64); 2950df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue LL = Op.getOperand(0); 2951df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue RL = Op.getOperand(1); 2952df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 2953df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 2954df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // UMUL_LOHI64 returns the low result in the odd register and the high 2955df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // result in the even register. SMUL_LOHI is defined to return the 2956df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // low half first, so the results are in reverse order. 2957df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2958df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford LL, RL, Ops[1], Ops[0]); 2959df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 2960df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 2961df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 2962df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 2963df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford } 2964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 2965df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford} 29661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 2967df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard SandifordSDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 2968df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SelectionDAG &DAG) const { 2969df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford EVT VT = Op.getValueType(); 2970df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford SDLoc DL(Op); 29711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[2]; 2972df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford if (is32Bit(VT)) 2973df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // Just do a normal 64-bit multiplication and extract the results. 2974df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // We define this so that it can be used for constant division. 2975df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 2976df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Op.getOperand(1), Ops[1], Ops[0]); 2977df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford else 2978df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // UMUL_LOHI64 returns the low result in the odd register and the high 2979df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // result in the even register. UMUL_LOHI is defined to return the 2980df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford // low half first, so the results are in reverse order. 2981df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2982df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 29841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 29851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 29861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 29871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 29881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Op0 = Op.getOperand(0); 29891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Op1 = Op.getOperand(1); 29901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT VT = Op.getValueType(); 2991ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 299235b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford unsigned Opcode; 29931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 29941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We use DSGF for 32-bit division. 29951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (is32Bit(VT)) { 29961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 299735b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford Opcode = SystemZISD::SDIVREM32; 299835b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford } else if (DAG.ComputeNumSignBits(Op1) > 32) { 299935b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 300035b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford Opcode = SystemZISD::SDIVREM32; 3001f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else 300235b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford Opcode = SystemZISD::SDIVREM64; 30031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // DSG(F) takes a 64-bit dividend, so the even register in the GR128 30051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // input is "don't care". The instruction returns the remainder in 30061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // the even register and the quotient in the odd register. 30071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[2]; 300835b7bebe1162326c38217ff80d4a49fbbffcc365Richard Sandiford lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 30091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op0, Op1, Ops[1], Ops[0]); 3010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 30111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 30121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 30141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 30151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT VT = Op.getValueType(); 3016ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 30171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // DL(G) uses a double-width dividend, so we need to clear the even 30191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // register in the GR128 input. The instruction returns the remainder 30201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // in the even register and the quotient in the odd register. 30211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[2]; 30221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (is32Bit(VT)) 30231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 30241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 30251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else 30261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 30271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 3028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 30291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 30301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 30321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 30331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the known-zero masks for each operand. 30351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 30361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand APInt KnownZero[2], KnownOne[2]; 3037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Ops[0], KnownZero[0], KnownOne[0]); 3038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Ops[1], KnownZero[1], KnownOne[1]); 30391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // See if the upper 32 bits of one operand and the lower 32 bits of the 30411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // other are known zero. They are the low and high operands respectively. 30421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand uint64_t Masks[] = { KnownZero[0].getZExtValue(), 30431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand KnownZero[1].getZExtValue() }; 30441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned High, Low; 30451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 30461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand High = 1, Low = 0; 30471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 30481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand High = 0, Low = 1; 30491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else 30501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 30511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue LowOp = Ops[Low]; 30531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue HighOp = Ops[High]; 30541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // If the high part is a constant, we're better off using IILH. 30561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (HighOp.getOpcode() == ISD::Constant) 30571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 30581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // If the low part is a constant that is outside the range of LHI, 30601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // then we're better off using IILF. 30611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (LowOp.getOpcode() == ISD::Constant) { 30621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 30631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (!isInt<16>(Value)) 30641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 30651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 30661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Check whether the high part is an AND that doesn't change the 30681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // high 32 bits and just masks out low bits. We can skip it if so. 30691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (HighOp.getOpcode() == ISD::AND && 30701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand HighOp.getOperand(1).getOpcode() == ISD::Constant) { 30712a2a323488a31fbdb3524f7f288b8e5c3fc3b7c3Richard Sandiford SDValue HighOp0 = HighOp.getOperand(0); 30722a2a323488a31fbdb3524f7f288b8e5c3fc3b7c3Richard Sandiford uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue(); 30732a2a323488a31fbdb3524f7f288b8e5c3fc3b7c3Richard Sandiford if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff)))) 30742a2a323488a31fbdb3524f7f288b8e5c3fc3b7c3Richard Sandiford HighOp = HighOp0; 30751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 30761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Take advantage of the fact that all GR32 operations only change the 30781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // low 32 bits by truncating Low to an i32 and inserting it directly 30791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // using a subreg. The interesting cases are those where the truncation 30801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // can be folded. 3081ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 30821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 3083745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, 30847c7b431d2fa3ead0a2a24c4dd81fc92f293203ddRichard Sandiford MVT::i64, HighOp, Low32); 30851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 30861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 30870c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerCTPOP(SDValue Op, 30880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SelectionDAG &DAG) const { 30890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar EVT VT = Op.getValueType(); 30900c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDLoc DL(Op); 30916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(0); 30926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 30936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle vector types via VPOPCT. 30946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT.isVector()) { 30956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op); 30966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op); 30976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch (VT.getVectorElementType().getSizeInBits()) { 30986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case 8: 30996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 31006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case 16: { 31016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); 31026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Shift = DAG.getConstant(8, DL, MVT::i32); 31036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift); 31046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp); 31056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift); 31066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 31076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 31086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case 32: { 31096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8, 31106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)); 31116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp); 31126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 31136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 31146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case 64: { 31156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8, 31166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)); 31176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp); 31186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp); 31196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 31206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 31216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar default: 31226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar llvm_unreachable("Unexpected type"); 31236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 31246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 31256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 31260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31270c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Get the known-zero mask for the operand. 31280c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar APInt KnownZero, KnownOne; 31290c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DAG.computeKnownBits(Op, KnownZero, KnownOne); 31300c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned NumSignificantBits = (~KnownZero).getActiveBits(); 31310c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (NumSignificantBits == 0) 31326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getConstant(0, DL, VT); 31330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Skip known-zero high parts of the operand. 31356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int64_t OrigBitSize = VT.getSizeInBits(); 31360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits); 31370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar BitSize = std::min(BitSize, OrigBitSize); 31380c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31390c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // The POPCNT instruction counts the number of bits in each byte. 31400c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op); 31410c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op); 31420c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); 31430c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31440c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Add up per-byte counts in a binary tree. All bits of Op at 31450c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // position larger than BitSize remain zero throughout. 31460c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar for (int64_t I = BitSize / 2; I >= 8; I = I / 2) { 31476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT)); 31480c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (BitSize != OrigBitSize) 31490c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp, 31506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT)); 31510c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp); 31520c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 31530c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31540c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Extract overall result from high byte. 31550c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (BitSize > 8) 31566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::SRL, DL, VT, Op, 31576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(BitSize - 8, DL, VT)); 31580c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 31590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return Op; 31600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 31610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 3162de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op, 3163de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) const { 3164de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Op); 3165de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>( 3166de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()); 3167de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SynchronizationScope FenceScope = static_cast<SynchronizationScope>( 3168de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue()); 3169de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3170de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The only fence that needs an instruction is a sequentially-consistent 3171de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // cross-thread fence. 3172de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (FenceOrdering == AtomicOrdering::SequentiallyConsistent && 3173de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar FenceScope == CrossThread) { 3174de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other, 3175de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op.getOperand(0)), 3176de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 0); 3177de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 3178de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3179de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // MEMBARRIER is a compiler barrier; it codegens to a no-op. 3180de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0)); 3181de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 3182de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 318336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Op is an atomic load. Lower it into a normal volatile load. 318436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 318536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 318636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<AtomicSDNode>(Op.getNode()); 318736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(), 318836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getChain(), Node->getBasePtr(), 318936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getMemoryVT(), Node->getMemOperand()); 319036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 319136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 319236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Op is an atomic store. Lower it into a normal volatile store followed 319336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// by a serialization. 319436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op, 319536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 319636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<AtomicSDNode>(Op.getNode()); 319736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(), 319836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getBasePtr(), Node->getMemoryVT(), 319936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getMemOperand()); 320036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other, 320136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Chain), 0); 320236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 320336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 32041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 32051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 320636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, 320736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG, 320836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Opcode) const { 320936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<AtomicSDNode>(Op.getNode()); 32101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // 32-bit operations need no code outside the main loop. 32121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT NarrowVT = Node->getMemoryVT(); 32131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT WideVT = MVT::i32; 32141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (NarrowVT == WideVT) 32151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 32161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t BitSize = NarrowVT.getSizeInBits(); 32181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ChainIn = Node->getChain(); 32191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Addr = Node->getBasePtr(); 32201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Src2 = Node->getVal(); 32211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineMemOperand *MMO = Node->getMemOperand(); 3222ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Node); 32231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT PtrVT = Addr.getValueType(); 32241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Convert atomic subtracts of constants into additions. 32261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 322736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) { 32281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Opcode = SystemZISD::ATOMIC_LOADW_ADD; 32296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType()); 32301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 32311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the address of the containing word. 32331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 32346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-4, DL, PtrVT)); 32351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the number of bits that the word must be rotated left in order 32371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // to bring the field to the top bits of a GR32. 32381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 32396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, PtrVT)); 32401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 32411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the complementing shift amount, for rotating a field in the top 32431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // bits back to its proper position. 32441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 32456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, WideVT), BitShift); 32461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Extend the source operand to 32 bits and prepare it for the inner loop. 32481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 32491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // operations require the source to be shifted in advance. (This shift 32501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // can be folded if the source is constant.) For AND and NAND, the lower 32511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // bits must be set, while for other opcodes they should be left clear. 32521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Opcode != SystemZISD::ATOMIC_SWAPW) 32531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 32546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32 - BitSize, DL, WideVT)); 32551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 32561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Opcode == SystemZISD::ATOMIC_LOADW_NAND) 32571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 32586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT)); 32591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Construct the ATOMIC_LOADW_* node. 32611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 32621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 32636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(BitSize, DL, WideVT) }; 32641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 32651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NarrowVT, MMO); 32661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Rotate the result of the final CS so that the field is in the lower 32681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // bits of a GR32, then truncate it. 32691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 32706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(BitSize, DL, WideVT)); 32711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 32721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 32731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 3274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(RetOps, DL); 32751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 32761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 327736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations 327836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit 327936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// operations into additions. 328036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op, 328136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 328236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<AtomicSDNode>(Op.getNode()); 328336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT MemVT = Node->getMemoryVT(); 328436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MemVT == MVT::i32 || MemVT == MVT::i64) { 328536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // A full-width operation. 328636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op.getValueType() == MemVT && "Mismatched VTs"); 328736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Src2 = Node->getVal(); 328836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue NegSrc2; 328936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Src2); 329036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 329136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) { 329236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use an addition if the operand is constant and either LAA(G) is 329336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // available or the negative value is in the range of A(G)FHI. 329436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t Value = (-Op2->getAPIntValue()).getSExtValue(); 3295c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1()) 32966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NegSrc2 = DAG.getConstant(Value, DL, MemVT); 3297c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } else if (Subtarget.hasInterlockedAccess1()) 329836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use LAA(G) if available. 32996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), 330036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Src2); 330136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 330236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NegSrc2.getNode()) 330336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, 330436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getChain(), Node->getBasePtr(), NegSrc2, 330536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getMemOperand(), Node->getOrdering(), 330636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Node->getSynchScope()); 330736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 330836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use the node as-is. 330936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Op; 331036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 331136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 331236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 331336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 331436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 33161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// into a fullword ATOMIC_CMP_SWAPW operation. 33171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 33181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 331936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<AtomicSDNode>(Op.getNode()); 33201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // We have native support for 32-bit compare and swap. 33221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT NarrowVT = Node->getMemoryVT(); 33231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT WideVT = MVT::i32; 33241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (NarrowVT == WideVT) 33251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return Op; 33261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand int64_t BitSize = NarrowVT.getSizeInBits(); 33281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue ChainIn = Node->getOperand(0); 33291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Addr = Node->getOperand(1); 33301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue CmpVal = Node->getOperand(2); 33311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue SwapVal = Node->getOperand(3); 33321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineMemOperand *MMO = Node->getMemOperand(); 3333ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Node); 33341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand EVT PtrVT = Addr.getValueType(); 33351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the address of the containing word. 33371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 33386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-4, DL, PtrVT)); 33391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the number of bits that the word must be rotated left in order 33411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // to bring the field to the top bits of a GR32. 33421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 33436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, PtrVT)); 33441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 33451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the complementing shift amount, for rotating a field in the top 33471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // bits back to its proper position. 33481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 33496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, WideVT), BitShift); 33501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Construct the ATOMIC_CMP_SWAPW node. 33521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 33531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 33546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NegBitShift, DAG.getConstant(BitSize, DL, WideVT) }; 33551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 3356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VTList, Ops, NarrowVT, MMO); 33571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return AtomicOp; 33581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 33591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 33611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 33621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 33631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 3364ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 33651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::R15D, Op.getValueType()); 33661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 33671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 33681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 33691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 33701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = DAG.getMachineFunction(); 33711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 3372de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool StoreBackchain = MF.getFunction()->hasFnAttribute("backchain"); 3373de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3374de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Chain = Op.getOperand(0); 3375de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue NewSP = Op.getOperand(1); 3376de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Backchain; 3377de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(Op); 3378de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3379de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (StoreBackchain) { 3380de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, MVT::i64); 3381de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo(), 3382de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, false, 0); 3383de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 3384de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3385de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R15D, NewSP); 3386de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3387de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (StoreBackchain) 3388de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo(), 3389de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false, 0); 3390de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 3391de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Chain; 33921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 33931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 3394a550b51bac50493db75a7b5788a3f2c3b62fd913Richard SandifordSDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op, 3395a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford SelectionDAG &DAG) const { 3396a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 3397a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford if (!IsData) 3398a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford // Just preserve the chain. 3399a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford return Op.getOperand(0); 3400a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford 34016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 3402a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 3403a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ; 340436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines auto *Node = cast<MemIntrinsicSDNode>(Op.getNode()); 3405a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford SDValue Ops[] = { 3406a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford Op.getOperand(0), 34076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Code, DL, MVT::i32), 3408a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford Op.getOperand(1) 3409a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford }; 34106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL, 3411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Node->getVTList(), Ops, 3412a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford Node->getMemoryVT(), Node->getMemOperand()); 3413a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford} 3414a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford 34150c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Return an i32 that contains the value of CC immediately after After, 34160c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// whose final operand must be MVT::Glue. 34170c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarstatic SDValue getCCResult(SelectionDAG &DAG, SDNode *After) { 34186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(After); 34190c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Glue = SDValue(After, After->getNumValues() - 1); 34206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue); 34216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM, 34226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32)); 34230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 34240c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 34250c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSDValue 34260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op, 34270c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SelectionDAG &DAG) const { 34280c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Opcode, CCValid; 34290c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) { 34300c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar assert(Op->getNumValues() == 2 && "Expected only CC result and chain"); 34310c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Glued = emitIntrinsicWithChainAndGlue(DAG, Op, Opcode); 34320c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue CC = getCCResult(DAG, Glued.getNode()); 34330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC); 34340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return SDValue(); 34350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 34360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 34370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return SDValue(); 34380c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 34390c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 34406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue 34416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op, 34426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 34436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode, CCValid; 34446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isIntrinsicWithCC(Op, Opcode, CCValid)) { 34456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Glued = emitIntrinsicWithGlue(DAG, Op, Opcode); 34466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC = getCCResult(DAG, Glued.getNode()); 34476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op->getNumValues() == 1) 34486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return CC; 34496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result"); 3450f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(), Glued, 3451f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CC); 34526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 34536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 34556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch (Id) { 3456de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case Intrinsic::thread_pointer: 3457de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return lowerThreadPointer(SDLoc(Op), DAG); 3458de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 34596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vpdi: 34606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(), 34616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 34626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vperm: 34646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(), 34656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 34666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuphb: 34686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuphh: 34696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuphf: 34706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(), 34716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1)); 34726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplhb: 34746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplhh: 34756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplhf: 34766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(), 34776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1)); 34786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplb: 34806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplhw: 34816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vuplf: 34826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(), 34836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1)); 34846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vupllb: 34866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vupllh: 34876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vupllf: 34886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(), 34896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1)); 34906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 34916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumb: 34926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumh: 34936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumgh: 34946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumgf: 34956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumqf: 34966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case Intrinsic::s390_vsumqg: 34976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(), 34986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2)); 34996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 35006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 35016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 35026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 35036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 35046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarnamespace { 35056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Says that SystemZISD operation Opcode can be used to perform the equivalent 35066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// of a VPERM with permute vector Bytes. If Opcode takes three operands, 35076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Operand is the constant third operand, otherwise it is the number of 35086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// bytes in each element of the result. 35096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstruct Permute { 35106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode; 35116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Operand; 35126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned char Bytes[SystemZ::VectorBytes]; 35136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar}; 35146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 35156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 35166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const Permute PermuteForms[] = { 35176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRHG 35186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_HIGH, 8, 35196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } }, 35206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRHF 35216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_HIGH, 4, 35226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } }, 35236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRHH 35246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_HIGH, 2, 35256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } }, 35266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRHB 35276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_HIGH, 1, 35286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } }, 35296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRLG 35306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_LOW, 8, 35316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } }, 35326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRLF 35336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_LOW, 4, 35346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } }, 35356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRLH 35366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_LOW, 2, 35376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } }, 35386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VMRLB 35396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::MERGE_LOW, 1, 35406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } }, 35416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPKG 35426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::PACK, 4, 35436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } }, 35446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPKF 35456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::PACK, 2, 35466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } }, 35476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPKH 35486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::PACK, 1, 35496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } }, 35506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPDI V1, V2, 4 (low half of V1, high half of V2) 35516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::PERMUTE_DWORDS, 4, 35526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } }, 35536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPDI V1, V2, 1 (high half of V1, low half of V2) 35546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { SystemZISD::PERMUTE_DWORDS, 1, 35556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } } 35566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar}; 35576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 35586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Called after matching a vector shuffle against a particular pattern. 35596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Both the original shuffle and the pattern have two vector operands. 35606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// OpNos[0] is the operand of the original shuffle that should be used for 35616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything. 35626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and 35636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// set OpNo0 and OpNo1 to the shuffle operands that should actually be used 35646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// for operands 0 and 1 of the pattern. 35656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) { 35666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNos[0] < 0) { 35676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNos[1] < 0) 35686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 35696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNo0 = OpNo1 = OpNos[1]; 35706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if (OpNos[1] < 0) { 35716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNo0 = OpNo1 = OpNos[0]; 35726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 35736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNo0 = OpNos[0]; 35746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNo1 = OpNos[1]; 35756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 35766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 35776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 35786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 35796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Bytes is a VPERM-like permute vector, except that -1 is used for 35806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// undefined bytes. Return true if the VPERM can be implemented using P. 35816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// When returning true set OpNo0 to the VPERM operand that should be 35826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// used for operand 0 of P and likewise OpNo1 for operand 1 of P. 35836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// 35846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// For example, if swapping the VPERM operands allows P to match, OpNo0 35856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one 35866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// operand, but rewriting it to use two duplicated operands allows it to 35876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// match P, then OpNo0 and OpNo1 will be the same. 35886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P, 35896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned &OpNo0, unsigned &OpNo1) { 35906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int OpNos[] = { -1, -1 }; 35916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) { 35926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Elt = Bytes[I]; 35936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Elt >= 0) { 35946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that the two permute vectors use the same suboperand 35956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // byte number. Only the operand numbers (the high bits) are 35966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // allowed to differ. 35976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1)) 35986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 35996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes; 36006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes; 36016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that the operand mappings are consistent with previous 36026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // elements. 36036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNos[ModelOpNo] == 1 - RealOpNo) 36046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 36056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNos[ModelOpNo] = RealOpNo; 36066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return chooseShuffleOpNos(OpNos, OpNo0, OpNo1); 36096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// As above, but search for a matching permute. 36126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const Permute *matchPermute(const SmallVectorImpl<int> &Bytes, 36136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned &OpNo0, unsigned &OpNo1) { 36146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (auto &P : PermuteForms) 36156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (matchPermute(Bytes, P, OpNo0, OpNo1)) 36166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return &P; 36176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return nullptr; 36186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Bytes is a VPERM-like permute vector, except that -1 is used for 36216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// undefined bytes. This permute is an operand of an outer permute. 36226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// See whether redistributing the -1 bytes gives a shuffle that can be 36236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// implemented using P. If so, set Transform to a VPERM-like permute vector 36246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// that, when applied to the result of P, gives the original permute in Bytes. 36256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool matchDoublePermute(const SmallVectorImpl<int> &Bytes, 36266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const Permute &P, 36276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVectorImpl<int> &Transform) { 36286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned To = 0; 36296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) { 36306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Elt = Bytes[From]; 36316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Elt < 0) 36326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Byte number From of the result is undefined. 36336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Transform[From] = -1; 36346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else { 36356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar while (P.Bytes[To] != Elt) { 36366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar To += 1; 36376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (To == SystemZ::VectorBytes) 36386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 36396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Transform[From] = To; 36416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 36446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// As above, but search for a matching permute. 36476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes, 36486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVectorImpl<int> &Transform) { 36496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (auto &P : PermuteForms) 36506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (matchDoublePermute(Bytes, P, Transform)) 36516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return &P; 36526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return nullptr; 36536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Convert the mask of the given VECTOR_SHUFFLE into a byte-level mask, 36566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// as if it had type vNi8. 36576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic void getVPermMask(ShuffleVectorSDNode *VSN, 36586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVectorImpl<int> &Bytes) { 36596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = VSN->getValueType(0); 36606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumElements = VT.getVectorNumElements(); 36616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = VT.getVectorElementType().getStoreSize(); 36626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes.resize(NumElements * BytesPerElement, -1); 36636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) { 36646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Index = VSN->getMaskElt(I); 36656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Index >= 0) 36666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned J = 0; J < BytesPerElement; ++J) 36676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J; 36686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Bytes is a VPERM-like permute vector, except that -1 is used for 36726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of 36736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// the result come from a contiguous sequence of bytes from one input. 36746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Set Base to the selector for the first byte if so. 36756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start, 36766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement, int &Base) { 36776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Base = -1; 36786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < BytesPerElement; ++I) { 36796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Bytes[Start + I] >= 0) { 36806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Elem = Bytes[Start + I]; 36816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Base < 0) { 36826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Base = Elem - I; 36836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure the bytes would come from one input operand. 36846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size()) 36856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 36866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if (unsigned(Base) != Elem - I) 36876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 36886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 36906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 36916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 36926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 36936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Bytes is a VPERM-like permute vector, except that -1 is used for 36946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// undefined bytes. Return true if it can be performed using VSLDI. 36956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// When returning true, set StartIndex to the shift amount and OpNo0 36966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// and OpNo1 to the VPERM operands that should be used as the first 36976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// and second shift operand respectively. 36986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes, 36996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned &StartIndex, unsigned &OpNo0, 37006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned &OpNo1) { 37016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int OpNos[] = { -1, -1 }; 37026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Shift = -1; 37036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < 16; ++I) { 37046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Index = Bytes[I]; 37056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Index >= 0) { 37066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int ExpectedShift = (Index - I) % SystemZ::VectorBytes; 37076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes; 37086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int RealOpNo = unsigned(Index) / SystemZ::VectorBytes; 37096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Shift < 0) 37106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Shift = ExpectedShift; 37116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Shift != ExpectedShift) 37126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 37136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that the operand mappings are consistent with previous 37146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // elements. 37156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNos[ModelOpNo] == 1 - RealOpNo) 37166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 37176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OpNos[ModelOpNo] = RealOpNo; 37186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 37196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 37206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar StartIndex = Shift; 37216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return chooseShuffleOpNos(OpNos, OpNo0, OpNo1); 37226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 37236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Create a node that performs P on operands Op0 and Op1, casting the 37256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// operands to the appropriate type. The type of the result is determined by P. 3726de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL, 37276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const Permute &P, SDValue Op0, SDValue Op1) { 37286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input 37296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // elements of a PACK are twice as wide as the outputs. 37306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 : 37316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar P.Opcode == SystemZISD::PACK ? P.Operand * 2 : 37326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar P.Operand); 37336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Cast both operands to the appropriate type. 37346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8), 37356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZ::VectorBytes / InBytes); 37366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0); 37376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1); 37386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op; 37396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (P.Opcode == SystemZISD::PERMUTE_DWORDS) { 37406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op2 = DAG.getConstant(P.Operand, DL, MVT::i32); 37416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2); 37426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if (P.Opcode == SystemZISD::PACK) { 37436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8), 37446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZ::VectorBytes / P.Operand); 37456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1); 37466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 37476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1); 37486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 37496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 37506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 37516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Bytes is a VPERM-like permute vector, except that -1 is used for 37536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// undefined bytes. Implement it on operands Ops[0] and Ops[1] using 37546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// VSLDI or VPERM. 3755de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL, 3756de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue *Ops, 37576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const SmallVectorImpl<int> &Bytes) { 37586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < 2; ++I) 37596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]); 37606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // First see whether VSLDI can be used. 37626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned StartIndex, OpNo0, OpNo1; 37636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1)) 37646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0], 37656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[OpNo1], DAG.getConstant(StartIndex, DL, MVT::i32)); 37666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Fall back on VPERM. Construct an SDNode for the permute vector. 37686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue IndexNodes[SystemZ::VectorBytes]; 37696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) 37706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Bytes[I] >= 0) 37716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32); 37726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 37736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IndexNodes[I] = DAG.getUNDEF(MVT::i32); 3774de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes); 37756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], Ops[1], Op2); 37766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 37776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarnamespace { 37796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Describes a general N-operand vector shuffle. 37806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstruct GeneralShuffle { 37816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GeneralShuffle(EVT vt) : VT(vt) {} 37826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar void addUndef(); 37836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar void add(SDValue, unsigned); 3784de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue getNode(SelectionDAG &, const SDLoc &); 37856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The operands of the shuffle. 37876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, SystemZ::VectorBytes> Ops; 37886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Index I is -1 if byte I of the result is undefined. Otherwise the 37906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand 37916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Bytes[I] / SystemZ::VectorBytes. 37926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<int, SystemZ::VectorBytes> Bytes; 37936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The type of the shuffle result. 37956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT; 37966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar}; 37976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 37986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 37996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Add an extra undefined element to the shuffle. 38006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarvoid GeneralShuffle::addUndef() { 38016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = VT.getVectorElementType().getStoreSize(); 38026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < BytesPerElement; ++I) 38036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes.push_back(-1); 38046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 38056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Add an extra element to the shuffle, taking it from element Elem of Op. 38076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// A null Op indicates a vector input whose value will be calculated later; 38086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// there is at most one such input per shuffle and it always has the same 38096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// type as the result. 38106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarvoid GeneralShuffle::add(SDValue Op, unsigned Elem) { 38116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = VT.getVectorElementType().getStoreSize(); 38126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The source vector can have wider elements than the result, 38146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // either through an explicit TRUNCATE or because of type legalization. 38156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // We want the least significant part. 38166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT FromVT = Op.getNode() ? Op.getValueType() : VT; 38176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize(); 38186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(FromBytesPerElement >= BytesPerElement && 38196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar "Invalid EXTRACT_VECTOR_ELT"); 38206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes + 38216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar (FromBytesPerElement - BytesPerElement)); 38226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Look through things like shuffles and bitcasts. 38246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar while (Op.getNode()) { 38256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getOpcode() == ISD::BITCAST) 38266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(0); 38276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) { 38286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether the bytes we need come from a contiguous part of one 38296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // operand. 38306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<int, SystemZ::VectorBytes> OpBytes; 38316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar getVPermMask(cast<ShuffleVectorSDNode>(Op), OpBytes); 38326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int NewByte; 38336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte)) 38346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 38356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (NewByte < 0) { 38366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addUndef(); 38376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return; 38386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 38396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes); 38406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Byte = unsigned(NewByte) % SystemZ::VectorBytes; 3841de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Op.isUndef()) { 38426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar addUndef(); 38436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return; 38446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else 38456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 38466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 38476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that the source of the extraction is in Ops. 38496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned OpNo = 0; 38506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (; OpNo < Ops.size(); ++OpNo) 38516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Ops[OpNo] == Op) 38526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 38536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNo == Ops.size()) 38546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(Op); 38556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Add the element to Bytes. 38576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Base = OpNo * SystemZ::VectorBytes + Byte; 38586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < BytesPerElement; ++I) 38596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes.push_back(Base + I); 38606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 38616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return SDNodes for the completed shuffle. 3863de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) { 38646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector"); 38656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Ops.size() == 0) 38676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getUNDEF(VT); 38686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that there are at least two shuffle operands. 38706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Ops.size() == 1) 38716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getUNDEF(MVT::v16i8)); 38726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Create a tree of shuffles, deferring root node until after the loop. 38746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Try to redistribute the undefined elements of non-root nodes so that 38756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // the non-root shuffles match something like a pack or merge, then adjust 38766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // the parent node's permute vector to compensate for the new order. 38776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Among other things, this copes with vectors like <2 x i16> that were 38786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // padded with undefined elements during type legalization. 38796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 38806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // In the best case this redistribution will lead to the whole tree 38816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // using packs and merges. It should rarely be a loss in other cases. 38826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Stride = 1; 38836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (; Stride * 2 < Ops.size(); Stride *= 2) { 38846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) { 38856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SubOps[] = { Ops[I], Ops[I + Stride] }; 38866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 38876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Create a mask for just these two operands. 38886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes); 38896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) { 38906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes; 38916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes; 38926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpNo == I) 38936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NewBytes[J] = Byte; 38946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (OpNo == I + Stride) 38956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NewBytes[J] = SystemZ::VectorBytes + Byte; 38966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 38976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NewBytes[J] = -1; 38986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 38996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See if it would be better to reorganize NewMask to avoid using VPERM. 39006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes); 39016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) { 39026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]); 39036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Applying NewBytesMap to Ops[I] gets back to NewBytes. 39046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) { 39056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (NewBytes[J] >= 0) { 39066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes && 39076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar "Invalid double permute"); 39086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J]; 39096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else 39106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(NewBytesMap[J] < 0 && "Invalid double permute"); 39116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 39136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Just use NewBytes on the operands. 39146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes); 39156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) 39166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (NewBytes[J] >= 0) 39176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes[J] = I * SystemZ::VectorBytes + J; 39186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Now we just have 2 inputs. Put the second operand in Ops[1]. 39236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Stride > 1) { 39246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[1] = Ops[Stride]; 39256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) 39266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Bytes[I] >= int(SystemZ::VectorBytes)) 39276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes; 39286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Look for an instruction that can do the permute without resorting 39316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // to VPERM. 39326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned OpNo0, OpNo1; 39336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op; 39346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1)) 39356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]); 39366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 39376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes); 39386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Op); 39396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 39406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion. 39426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool isScalarToVector(SDValue Op) { 39436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I) 3944de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Op.getOperand(I).isUndef()) 39456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 39466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 39476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 39486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return a vector of type VT that contains Value in the first element. 39506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// The other elements don't matter. 3951de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 39526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Value) { 39536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If we have a constant, replicate it to all elements and let the 39546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // BUILD_VECTOR lowering take care of it. 39556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Value.getOpcode() == ISD::Constant || 39566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value.getOpcode() == ISD::ConstantFP) { 39576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value); 3958de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getBuildVector(VT, DL, Ops); 39596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Value.isUndef()) 39616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getUNDEF(VT); 39626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value); 39636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 39646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return a vector of type VT in which Op0 is in element 0 and Op1 is in 39666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// element 1. Used for cases in which replication is cheap. 3967de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 39686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op0, SDValue Op1) { 3969de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op0.isUndef()) { 3970de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op1.isUndef()) 39716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getUNDEF(VT); 39726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1); 39736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 3974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op1.isUndef()) 39756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0); 39766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT, 39776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar buildScalarToVector(DAG, DL, VT, Op0), 39786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar buildScalarToVector(DAG, DL, VT, Op1)); 39796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 39806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 39816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64 39826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// vector for them. 3983de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0, 39846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op1) { 3985de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op0.isUndef() && Op1.isUndef()) 39866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getUNDEF(MVT::v2i64); 39876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If one of the two inputs is undefined then replicate the other one, 39886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // in order to avoid using another register unnecessarily. 3989de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op0.isUndef()) 39906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1); 3991de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else if (Op1.isUndef()) 39926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); 39936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else { 39946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); 39956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1); 39966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 39976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1); 39986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 39996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 40006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Try to represent constant BUILD_VECTOR node BVN using a 40016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// SystemZISD::BYTE_MASK-style mask. Store the mask value in Mask 40026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// on success. 40036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask) { 40046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT ElemVT = BVN->getValueType(0).getVectorElementType(); 40056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = ElemVT.getStoreSize(); 40066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0, E = BVN->getNumOperands(); I != E; ++I) { 40076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = BVN->getOperand(I); 4008de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Op.isUndef()) { 40096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Value; 40106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getOpcode() == ISD::Constant) 40116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value = dyn_cast<ConstantSDNode>(Op)->getZExtValue(); 40126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Op.getOpcode() == ISD::ConstantFP) 40136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value = (dyn_cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt() 40146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar .getZExtValue()); 40156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 40166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 40176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned J = 0; J < BytesPerElement; ++J) { 40186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Byte = (Value >> (J * 8)) & 0xff; 40196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Byte == 0xff) 40206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Mask |= 1ULL << ((E - I - 1) * BytesPerElement + J); 40216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Byte != 0) 40226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 40236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 40276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 40286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 40296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Try to load a vector constant in which BitsPerElement-bit value Value 40306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// is replicated to fill the vector. VT is the type of the resulting 40316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// constant, which may have elements of a different size from BitsPerElement. 40326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return the SDValue of the constant on success, otherwise return 40336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// an empty value. 40346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic SDValue tryBuildVectorReplicate(SelectionDAG &DAG, 40356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const SystemZInstrInfo *TII, 4036de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, EVT VT, uint64_t Value, 40376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BitsPerElement) { 40386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Signed 16-bit values can be replicated using VREPI. 40396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int64_t SignedValue = SignExtend64(Value, BitsPerElement); 40406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isInt<16>(SignedValue)) { 40416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), 40426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZ::VectorBits / BitsPerElement); 40436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = DAG.getNode(SystemZISD::REPLICATE, DL, VecVT, 40446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(SignedValue, DL, MVT::i32)); 40456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Op); 40466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether rotating the constant left some N places gives a value that 40486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // is one less than a power of 2 (i.e. all zeros followed by all ones). 40496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If so we can use VGM. 40506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Start, End; 40516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (TII->isRxSBGMask(Value, BitsPerElement, Start, End)) { 40526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // isRxSBGMask returns the bit numbers for a full 64-bit value, 40536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // with 0 denoting 1 << 63 and 63 denoting 1. Convert them to 40546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // bit numbers for an BitsPerElement value, so that 0 denotes 40556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 1 << (BitsPerElement-1). 40566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Start -= 64 - BitsPerElement; 40576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar End -= 64 - BitsPerElement; 40586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), 40596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZ::VectorBits / BitsPerElement); 40606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = DAG.getNode(SystemZISD::ROTATE_MASK, DL, VecVT, 40616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Start, DL, MVT::i32), 40626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(End, DL, MVT::i32)); 40636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Op); 40646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 40666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 40676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 40686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually 40696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for 40706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR 40716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// would benefit from this representation and return it if so. 40726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic SDValue tryBuildVectorShuffle(SelectionDAG &DAG, 40736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar BuildVectorSDNode *BVN) { 40746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = BVN->getValueType(0); 40756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumElements = VT.getVectorNumElements(); 40766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 40776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation 40786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still 40796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // need a BUILD_VECTOR, add an additional placeholder operand for that 40806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // BUILD_VECTOR and store its operands in ResidueOps. 40816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GeneralShuffle GS(VT); 40826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps; 40836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool FoundOne = false; 40846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) { 40856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = BVN->getOperand(I); 40866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getOpcode() == ISD::TRUNCATE) 40876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(0); 40886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 40896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(1).getOpcode() == ISD::Constant) { 40906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Elem = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 40916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GS.add(Op.getOperand(0), Elem); 40926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar FoundOne = true; 4093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Op.isUndef()) { 40946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GS.addUndef(); 40956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 40966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GS.add(SDValue(), ResidueOps.size()); 4097f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResidueOps.push_back(BVN->getOperand(I)); 40986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 40996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Nothing to do if there are no EXTRACT_VECTOR_ELTs. 41026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!FoundOne) 41036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 41046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Create the BUILD_VECTOR for the remaining elements, if any. 41066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!ResidueOps.empty()) { 41076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar while (ResidueOps.size() < NumElements) 4108f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType())); 41096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (auto &Op : GS.Ops) { 41106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Op.getNode()) { 4111de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps); 41126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 41136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return GS.getNode(DAG, SDLoc(BVN)); 41176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 41186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Combine GPR scalar values Elems into a vector of type VT. 4120de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 41216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVectorImpl<SDValue> &Elems) { 41226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether there is a single replicated value. 41236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Single; 41246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned int NumElements = Elems.size(); 41256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned int Count = 0; 41266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (auto Elem : Elems) { 4127de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Elem.isUndef()) { 41286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Single.getNode()) 41296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Single = Elem; 41306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Elem != Single) { 41316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Single = SDValue(); 41326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 41336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Count += 1; 41356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // There are three cases here: 41386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 41396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // - if the only defined element is a loaded one, the best sequence 41406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // is a replicating load. 41416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 41426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // - otherwise, if the only defined element is an i64 value, we will 41436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // end up with the same VLVGP sequence regardless of whether we short-cut 41446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // for replication or fall through to the later code. 41456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 41466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // - otherwise, if the only defined element is an i32 or smaller value, 41476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // we would need 2 instructions to replicate it: VLVGP followed by VREPx. 41486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // This is only a win if the single defined element is used more than once. 41496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // In other cases we're better off using a single VLVGx. 41506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Single.getNode() && (Count > 1 || Single.getOpcode() == ISD::LOAD)) 41516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single); 41526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The best way of building a v2i64 from two i64s is to use VLVGP. 41546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT == MVT::v2i64) 41556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return joinDwords(DAG, DL, Elems[0], Elems[1]); 41566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Use a 64-bit merge high to combine two doubles. 41586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT == MVT::v2f64) 41596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]); 41606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Build v4f32 values directly from the FPRs: 41626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 41636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // <Axxx> <Bxxx> <Cxxxx> <Dxxx> 41646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // V V VMRHF 41656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // <ABxx> <CDxx> 41666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // V VMRHG 41676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // <ABCD> 41686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT == MVT::v4f32) { 41696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]); 41706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]); 41716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Avoid unnecessary undefs by reusing the other operand. 4172de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op01.isUndef()) 41736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op01 = Op23; 4174de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else if (Op23.isUndef()) 41756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op23 = Op01; 41766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Merging identical replications is a no-op. 41776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23) 41786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op01; 41796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01); 41806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23); 41816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH, 41826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DL, MVT::v2i64, Op01, Op23); 41836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Op); 41846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Collect the constant terms. 41876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue()); 41886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false); 41896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 41906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumConstants = 0; 41916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) { 41926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Elem = Elems[I]; 41936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Elem.getOpcode() == ISD::Constant || 41946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Elem.getOpcode() == ISD::ConstantFP) { 41956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NumConstants += 1; 41966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Constants[I] = Elem; 41976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Done[I] = true; 41986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 41996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 42006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If there was at least one constant, fill in the other elements of 42016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Constants with undefs to get a full vector constant and use that 42026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // as the starting point. 42036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Result; 42046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (NumConstants > 0) { 42056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) 42066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Constants[I].getNode()) 42076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Constants[I] = DAG.getUNDEF(Elems[I].getValueType()); 4208de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Result = DAG.getBuildVector(VT, DL, Constants); 42096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 42106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise try to use VLVGP to start the sequence in order to 42116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // avoid a false dependency on any previous contents of the vector 42126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // register. This only makes sense if one of the associated elements 42136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // is defined. 42146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned I1 = NumElements / 2 - 1; 42156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned I2 = NumElements - 1; 4216de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool Def1 = !Elems[I1].isUndef(); 4217de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool Def2 = !Elems[I2].isUndef(); 42186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Def1 || Def2) { 42196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Elem1 = Elems[Def1 ? I1 : I2]; 42206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Elem2 = Elems[Def2 ? I2 : I1]; 42216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Result = DAG.getNode(ISD::BITCAST, DL, VT, 42226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar joinDwords(DAG, DL, Elem1, Elem2)); 42236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Done[I1] = true; 42246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Done[I2] = true; 42256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else 42266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Result = DAG.getUNDEF(VT); 42276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 42286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Use VLVGx to insert the other elements. 42306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) 4231de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Done[I] && !Elems[I].isUndef()) 42326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I], 42336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(I, DL, MVT::i32)); 42346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Result; 42356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 42366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op, 42386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 42396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const SystemZInstrInfo *TII = 42406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 42416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar auto *BVN = cast<BuildVectorSDNode>(Op.getNode()); 42426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 42436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 42446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (BVN->isConstant()) { 42466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Try using VECTOR GENERATE BYTE MASK. This is the architecturally- 42476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // preferred way of creating all-zero and all-one vectors so give it 42486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // priority over other methods below. 42496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Mask = 0; 42506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (tryBuildVectorByteMask(BVN, Mask)) { 42516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8, 42526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Mask, DL, MVT::i32)); 42536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Op); 42546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 42556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Try using some form of replication. 42576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar APInt SplatBits, SplatUndef; 42586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned SplatBitSize; 42596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool HasAnyUndefs; 42606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 42616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8, true) && 42626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SplatBitSize <= 64) { 42636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // First try assuming that any undefined bits above the highest set bit 42646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // and below the lowest set bit are 1s. This increases the likelihood of 42656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // being able to use a sign-extended element value in VECTOR REPLICATE 42666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK. 42676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t SplatBitsZ = SplatBits.getZExtValue(); 42686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t SplatUndefZ = SplatUndef.getZExtValue(); 42696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Lower = (SplatUndefZ 42706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar & ((uint64_t(1) << findFirstSet(SplatBitsZ)) - 1)); 42716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Upper = (SplatUndefZ 42726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar & ~((uint64_t(1) << findLastSet(SplatBitsZ)) - 1)); 42736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Value = SplatBitsZ | Upper | Lower; 42746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value, 42756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SplatBitSize); 42766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getNode()) 42776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 42786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Now try assuming that any undefined bits between the first and 42806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // last defined set bits are set. This increases the chances of 42816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // using a non-wraparound mask. 42826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Middle = SplatUndefZ & ~Upper & ~Lower; 42836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value = SplatBitsZ | Middle; 42846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value, SplatBitSize); 42856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getNode()) 42866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 42876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 42886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Fall back to loading it from memory. 42906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 42916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 42926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See if we should use shuffles to construct the vector from other vectors. 4294de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Res = tryBuildVectorShuffle(DAG, BVN)) 42956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Res; 42966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 42976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Detect SCALAR_TO_VECTOR conversions. 42986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isOperationLegal(ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op)) 42996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return buildScalarToVector(DAG, DL, VT, Op.getOperand(0)); 43006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise use buildVector to build the vector up from GPRs. 43026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumElements = Op.getNumOperands(); 43036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements); 43046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) 43056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[I] = Op.getOperand(I); 43066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return buildVector(DAG, DL, VT, Ops); 43076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 43086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, 43106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 43116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode()); 43126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 43136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 43146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NumElements = VT.getVectorNumElements(); 43156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VSN->isSplat()) { 43176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op0 = Op.getOperand(0); 43186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Index = VSN->getSplatIndex(); 43196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(Index < VT.getVectorNumElements() && 43206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar "Splat index should be defined and in first operand"); 43216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether the value we're splatting is directly available as a scalar. 43226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) || 43236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op0.getOpcode() == ISD::BUILD_VECTOR) 43246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index)); 43256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise keep it as a vector-to-vector operation. 43266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0), 43276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Index, DL, MVT::i32)); 43286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 43296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GeneralShuffle GS(VT); 43316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned I = 0; I < NumElements; ++I) { 43326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int Elt = VSN->getMaskElt(I); 43336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Elt < 0) 43346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GS.addUndef(); 43356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else 43366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar GS.add(Op.getOperand(unsigned(Elt) / NumElements), 43376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned(Elt) % NumElements); 43386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 43396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return GS.getNode(DAG, SDLoc(VSN)); 43406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 43416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op, 43436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 43446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 43456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Just insert the scalar into element 0 of an undefined vector. 43466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, 43476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getValueType(), DAG.getUNDEF(Op.getValueType()), 43486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32)); 43496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 43506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op, 43526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 43536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle insertions of floating-point values. 43546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 43556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op0 = Op.getOperand(0); 43566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op1 = Op.getOperand(1); 43576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op2 = Op.getOperand(2); 43586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 43596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Insertions into constant indices of a v2f64 can be done using VPDI. 43616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // However, if the inserted value is a bitcast or a constant then it's 43626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // better to use GPRs, as below. 43636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT == MVT::v2f64 && 43646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op1.getOpcode() != ISD::BITCAST && 43656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op1.getOpcode() != ISD::ConstantFP && 43666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op2.getOpcode() == ISD::Constant) { 43676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Index = dyn_cast<ConstantSDNode>(Op2)->getZExtValue(); 43686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Mask = VT.getVectorNumElements() - 1; 43696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Index <= Mask) 43706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 43716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 43726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise bitcast to the equivalent integer form and insert via a GPR. 43746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT IntVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits()); 43756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements()); 43766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT, 43776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), 43786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2); 43796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Res); 43806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 43816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue 43836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op, 43846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG) const { 43856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Handle extractions of floating-point values. 43866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 43876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op0 = Op.getOperand(0); 43886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op1 = Op.getOperand(1); 43896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 43906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VecVT = Op0.getValueType(); 43916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 43926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Extractions of constant indices can be done directly. 43936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) { 43946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar uint64_t Index = CIndexN->getZExtValue(); 43956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Mask = VecVT.getVectorNumElements() - 1; 43966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Index <= Mask) 43976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 43986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 43996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise bitcast to the equivalent integer form and extract via a GPR. 44016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits()); 44026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements()); 44036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT, 44046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1); 44056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, DL, VT, Res); 44066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 44076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue 44096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG, 4410f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned UnpackHigh) const { 44116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue PackedOp = Op.getOperand(0); 44126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT OutVT = Op.getValueType(); 44136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT InVT = PackedOp.getValueType(); 44146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned ToBits = OutVT.getVectorElementType().getSizeInBits(); 44156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned FromBits = InVT.getVectorElementType().getSizeInBits(); 44166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar do { 44176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar FromBits *= 2; 44186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits), 44196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SystemZ::VectorBits / FromBits); 44206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar PackedOp = DAG.getNode(UnpackHigh, SDLoc(PackedOp), OutVT, PackedOp); 44216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } while (FromBits != ToBits); 44226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return PackedOp; 44236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 44246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG, 44266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned ByScalar) const { 44276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Look for cases where a vector shift can use the *_BY_SCALAR form. 44286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op0 = Op.getOperand(0); 44296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Op1 = Op.getOperand(1); 44306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 44316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 44326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned ElemBitSize = VT.getVectorElementType().getSizeInBits(); 44336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether the shift vector is a splat represented as BUILD_VECTOR. 44356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) { 44366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar APInt SplatBits, SplatUndef; 44376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned SplatBitSize; 44386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool HasAnyUndefs; 44396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Check for constant splats. Use ElemBitSize as the minimum element 44406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // width and reject splats that need wider elements. 44416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 44426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar ElemBitSize, true) && 44436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SplatBitSize == ElemBitSize) { 44446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff, 44456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DL, MVT::i32); 44466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ByScalar, DL, VT, Op0, Shift); 44476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Check for variable splats. 44496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar BitVector UndefElements; 44506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Splat = BVN->getSplatValue(&UndefElements); 44516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Splat) { 44526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Since i32 is the smallest legal type, we either need a no-op 44536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // or a truncation. 44546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat); 44556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ByScalar, DL, VT, Op0, Shift); 44566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // See whether the shift vector is a splat represented as SHUFFLE_VECTOR, 44606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // and the shift amount is directly available in a GPR. 44616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) { 44626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VSN->isSplat()) { 44636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue VSNOp0 = VSN->getOperand(0); 44646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Index = VSN->getSplatIndex(); 44656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(Index < VT.getVectorNumElements() && 44666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar "Splat index should be defined and in first operand"); 44676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) || 44686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VSNOp0.getOpcode() == ISD::BUILD_VECTOR) { 44696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Since i32 is the smallest legal type, we either need a no-op 44706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // or a truncation. 44716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, 44726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VSNOp0.getOperand(Index)); 44736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ByScalar, DL, VT, Op0, Shift); 44746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 44776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise just treat the current form as legal. 44796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 44806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 44816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 44821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandSDValue SystemZTargetLowering::LowerOperation(SDValue Op, 44831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SelectionDAG &DAG) const { 44841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand switch (Op.getOpcode()) { 4485de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::FRAMEADDR: 4486de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return lowerFRAMEADDR(Op, DAG); 4487de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::RETURNADDR: 4488de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return lowerRETURNADDR(Op, DAG); 44891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::BR_CC: 44901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerBR_CC(Op, DAG); 44911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SELECT_CC: 44921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerSELECT_CC(Op, DAG); 4493d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford case ISD::SETCC: 4494d1a4f579bf45aec933c79292b6b9663581438738Richard Sandiford return lowerSETCC(Op, DAG); 44951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::GlobalAddress: 44961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 44971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::GlobalTLSAddress: 44981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 44991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::BlockAddress: 45001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 45011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::JumpTable: 45021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 45031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ConstantPool: 45041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 45051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::BITCAST: 45061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerBITCAST(Op, DAG); 45071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::VASTART: 45081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerVASTART(Op, DAG); 45091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::VACOPY: 45101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerVACOPY(Op, DAG); 45111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::DYNAMIC_STACKALLOC: 45121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerDYNAMIC_STACKALLOC(Op, DAG); 4513de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::GET_DYNAMIC_AREA_OFFSET: 4514de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); 4515df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford case ISD::SMUL_LOHI: 4516df40f8e8ad0aa93defa44b8a136e8d871cfd44eaRichard Sandiford return lowerSMUL_LOHI(Op, DAG); 45171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::UMUL_LOHI: 45181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerUMUL_LOHI(Op, DAG); 45191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::SDIVREM: 45201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerSDIVREM(Op, DAG); 45211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::UDIVREM: 45221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerUDIVREM(Op, DAG); 45231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::OR: 45241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerOR(Op, DAG); 45250c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case ISD::CTPOP: 45260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return lowerCTPOP(Op, DAG); 4527de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::ATOMIC_FENCE: 4528de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return lowerATOMIC_FENCE(Op, DAG); 45291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_SWAP: 453036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW); 453136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::ATOMIC_STORE: 453236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_STORE(Op, DAG); 453336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::ATOMIC_LOAD: 453436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD(Op, DAG); 45351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_ADD: 453636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 45371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_SUB: 453836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_SUB(Op, DAG); 45391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_AND: 454036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 45411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_OR: 454236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 45431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_XOR: 454436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 45451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_NAND: 454636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 45471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_MIN: 454836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 45491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_MAX: 455036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 45511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_UMIN: 455236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 45531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_LOAD_UMAX: 455436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 45551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::ATOMIC_CMP_SWAP: 45561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerATOMIC_CMP_SWAP(Op, DAG); 45571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::STACKSAVE: 45581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerSTACKSAVE(Op, DAG); 45591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case ISD::STACKRESTORE: 45601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return lowerSTACKRESTORE(Op, DAG); 4561a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford case ISD::PREFETCH: 4562a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford return lowerPREFETCH(Op, DAG); 45630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case ISD::INTRINSIC_W_CHAIN: 45640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return lowerINTRINSIC_W_CHAIN(Op, DAG); 45656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::INTRINSIC_WO_CHAIN: 45666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerINTRINSIC_WO_CHAIN(Op, DAG); 45676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::BUILD_VECTOR: 45686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerBUILD_VECTOR(Op, DAG); 45696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::VECTOR_SHUFFLE: 45706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerVECTOR_SHUFFLE(Op, DAG); 45716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SCALAR_TO_VECTOR: 45726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerSCALAR_TO_VECTOR(Op, DAG); 45736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::INSERT_VECTOR_ELT: 45746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerINSERT_VECTOR_ELT(Op, DAG); 45756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::EXTRACT_VECTOR_ELT: 45766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerEXTRACT_VECTOR_ELT(Op, DAG); 45776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SIGN_EXTEND_VECTOR_INREG: 45786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACK_HIGH); 45796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::ZERO_EXTEND_VECTOR_INREG: 45806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACKL_HIGH); 45816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SHL: 45826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR); 45836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SRL: 45846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR); 45856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SRA: 45866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR); 45871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 45881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unexpected node to lower"); 45891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 45901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 45911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 45921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandconst char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 45931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 45946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch ((SystemZISD::NodeType)Opcode) { 45956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case SystemZISD::FIRST_NUMBER: break; 45961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(RET_FLAG); 45971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(CALL); 459880f54784da0bd42fb79176bbf447a31d69287fe3Richard Sandiford OPCODE(SIBCALL); 45996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(TLS_GDCALL); 46006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(TLS_LDCALL); 46011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(PCREL_WRAPPER); 46028dac19c0708c9bd0da0b832014918e00ded44d86Richard Sandiford OPCODE(PCREL_OFFSET); 460336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OPCODE(IABS); 4604aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford OPCODE(ICMP); 4605aff1c6427ce22125adfa29de4145030aa3214a2eRichard Sandiford OPCODE(FCMP); 4606477168192c98e1f75a5bc6db3d34a177f327bd34Richard Sandiford OPCODE(TM); 46071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(BR_CCMASK); 46081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(SELECT_CCMASK); 46091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ADJDYNALLOC); 46101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(EXTRACT_ACCESS); 46116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(POPCNT); 46121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(UMUL_LOHI64); 46136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(SDIVREM32); 46141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(SDIVREM64); 46151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(UDIVREM32); 46161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(UDIVREM64); 4617dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford OPCODE(MVC); 4618842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford OPCODE(MVC_LOOP); 461916277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(NC); 462016277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(NC_LOOP); 462116277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(OC); 462216277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(OC_LOOP); 462316277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(XC); 462416277c4698f36a756c540fae326874774156aaedRichard Sandiford OPCODE(XC_LOOP); 4625e03a56d62fc623e2f72d623b816f91b293d5904bRichard Sandiford OPCODE(CLC); 4626842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford OPCODE(CLC_LOOP); 46274fc7355a21e1fa838406e15459aaf54a58fcf909Richard Sandiford OPCODE(STPCPY); 46286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(STRCMP); 462919262ee0725a09b7c621a3d2eb66ba1513ae932aRichard Sandiford OPCODE(SEARCH_STRING); 4630ac168b8bc8773a083a10902f64e4ae57a925aee4Richard Sandiford OPCODE(IPM); 463136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OPCODE(SERIALIZE); 4632de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OPCODE(MEMBARRIER); 46330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar OPCODE(TBEGIN); 46340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar OPCODE(TBEGIN_NOFLOAT); 46350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar OPCODE(TEND); 46366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(BYTE_MASK); 46376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(ROTATE_MASK); 46386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(REPLICATE); 46396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(JOIN_DWORDS); 46406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(SPLAT); 46416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(MERGE_HIGH); 46426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(MERGE_LOW); 46436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(SHL_DOUBLE); 46446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(PERMUTE_DWORDS); 46456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(PERMUTE); 46466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(PACK); 46476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(PACKS_CC); 46486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(PACKLS_CC); 46496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(UNPACK_HIGH); 46506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(UNPACKL_HIGH); 46516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(UNPACK_LOW); 46526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(UNPACKL_LOW); 46536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSHL_BY_SCALAR); 46546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSRL_BY_SCALAR); 46556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSRA_BY_SCALAR); 46566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSUM); 46576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPE); 46586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPH); 46596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPHL); 46606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPES); 46616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPHS); 46626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VICMPHLS); 46636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPE); 46646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPH); 46656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPHE); 46666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPES); 46676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPHS); 46686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFCMPHES); 46696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFTCI); 46706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VEXTEND); 46716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VROUND); 46726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VTM); 46736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFAE_CC); 46746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFAEZ_CC); 46756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFEE_CC); 46766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFEEZ_CC); 46776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFENE_CC); 46786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VFENEZ_CC); 46796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VISTR_CC); 46806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSTRC_CC); 46816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OPCODE(VSTRCZ_CC); 4682de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OPCODE(TDC); 46831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_SWAPW); 46841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_ADD); 46851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_SUB); 46861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_AND); 46871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_OR); 46881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_XOR); 46891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_NAND); 46901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_MIN); 46911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_MAX); 46921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_UMIN); 46931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_LOADW_UMAX); 46941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand OPCODE(ATOMIC_CMP_SWAPW); 4695de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OPCODE(LRV); 4696de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OPCODE(STRV); 4697a550b51bac50493db75a7b5788a3f2c3b62fd913Richard Sandiford OPCODE(PREFETCH); 46981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 4699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 47001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand#undef OPCODE 47011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 47021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 47036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Return true if VT is a vector whose elements are a whole number of bytes 47046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// in width. 47056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic bool canTreatAsByteVector(EVT VT) { 47066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return VT.isVector() && VT.getVectorElementType().getSizeInBits() % 8 == 0; 47076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 47086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 47096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT 47106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// producing a result of type ResVT. Op is a possibly bitcast version 47116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// of the input vector and Index is the index (based on type VecVT) that 47126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// should be extracted. Return the new extraction if a simplification 47136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// was possible or if Force is true. 4714de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT, 4715de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VecVT, SDValue Op, 4716de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Index, 47176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAGCombinerInfo &DCI, 47186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool Force) const { 47196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 47206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 47216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The number of bytes being extracted. 47226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize(); 47236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 47246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (;;) { 47256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode = Op.getOpcode(); 47266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Opcode == ISD::BITCAST) 47276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Look through bitcasts. 47286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(0); 47296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar else if (Opcode == ISD::VECTOR_SHUFFLE && 47306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar canTreatAsByteVector(Op.getValueType())) { 47316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Get a VPERM-like permute mask and see whether the bytes covered 47326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // by the extracted element are a contiguous sequence from one 47336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // source operand. 47346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SmallVector<int, SystemZ::VectorBytes> Bytes; 47356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar getVPermMask(cast<ShuffleVectorSDNode>(Op), Bytes); 47366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int First; 47376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!getShuffleInput(Bytes, Index * BytesPerElement, 47386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar BytesPerElement, First)) 47396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 47406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (First < 0) 47416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getUNDEF(ResVT); 47426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure the contiguous sequence starts at a multiple of the 47436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // original element size. 47446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Byte = unsigned(First) % Bytes.size(); 47456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Byte % BytesPerElement != 0) 47466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 47476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // We can get the extracted value directly from an input. 47486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Index = Byte / BytesPerElement; 47496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(unsigned(First) / Bytes.size()); 47506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Force = true; 47516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if (Opcode == ISD::BUILD_VECTOR && 47526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar canTreatAsByteVector(Op.getValueType())) { 47536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // We can only optimize this case if the BUILD_VECTOR elements are 47546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // at least as wide as the extracted value. 47556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT OpVT = Op.getValueType(); 47566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize(); 47576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (OpBytesPerElement < BytesPerElement) 47586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 47596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that the least-significant bit of the extracted value 47606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // is the least significant bit of an input. 47616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned End = (Index + 1) * BytesPerElement; 47626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (End % OpBytesPerElement != 0) 47636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 47646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // We're extracting the low part of one operand of the BUILD_VECTOR. 47656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(End / OpBytesPerElement - 1); 47666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Op.getValueType().isInteger()) { 47676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = MVT::getIntegerVT(Op.getValueType().getSizeInBits()); 47686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); 47696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DCI.AddToWorklist(Op.getNode()); 47706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 47716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = MVT::getIntegerVT(ResVT.getSizeInBits()); 47726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); 47736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (VT != ResVT) { 47746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DCI.AddToWorklist(Op.getNode()); 47756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op); 47766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 47776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Op; 47786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || 4779f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode == ISD::ZERO_EXTEND_VECTOR_INREG || 4780f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode == ISD::ANY_EXTEND_VECTOR_INREG) && 4781f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar canTreatAsByteVector(Op.getValueType()) && 47826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar canTreatAsByteVector(Op.getOperand(0).getValueType())) { 47836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure that only the unextended bits are significant. 47846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT ExtVT = Op.getValueType(); 47856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT OpVT = Op.getOperand(0).getValueType(); 47866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize(); 47876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize(); 47886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Byte = Index * BytesPerElement; 47896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned SubByte = Byte % ExtBytesPerElement; 47906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement; 47916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (SubByte < MinSubByte || 4792f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SubByte + BytesPerElement > ExtBytesPerElement) 4793f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 47946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Get the byte offset of the unextended element 47956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Byte = Byte / ExtBytesPerElement * OpBytesPerElement; 47966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // ...then add the byte offset relative to that element. 47976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Byte += SubByte - MinSubByte; 47986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Byte % BytesPerElement != 0) 4799f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 48006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = Op.getOperand(0); 48016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Index = Byte / BytesPerElement; 48026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Force = true; 48036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else 48046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 48056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Force) { 48076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getValueType() != VecVT) { 48086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op); 48096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DCI.AddToWorklist(Op.getNode()); 48106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op, 48126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Index, DL, MVT::i32)); 48136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 48156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 48166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 48176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// Optimize vector operations in scalar value Op on the basis that Op 48186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar// is truncated to TruncVT. 4819de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineTruncateExtract( 4820de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const { 48216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If we have (trunc (extract_vector_elt X, Y)), try to turn it into 48226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements 48236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // of type TruncVT. 48246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 48256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar TruncVT.getSizeInBits() % 8 == 0) { 48266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Vec = Op.getOperand(0); 48276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VecVT = Vec.getValueType(); 48286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (canTreatAsByteVector(VecVT)) { 48296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 48306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize(); 48316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned TruncBytes = TruncVT.getStoreSize(); 48326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (BytesPerElement % TruncBytes == 0) { 48336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Calculate the value of Y' in the above description. We are 48346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // splitting the original elements into Scale equal-sized pieces 48356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // and for truncation purposes want the last (least-significant) 48366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // of these pieces for IndexN. This is easiest to do by calculating 48376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // the start index of the following element and then subtracting 1. 48386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Scale = BytesPerElement / TruncBytes; 48396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1; 48406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 48416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Defer the creation of the bitcast from X to combineExtract, 48426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // which might be able to optimize the extraction. 48436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VecVT = MVT::getVectorVT(MVT::getIntegerVT(TruncBytes * 8), 48446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VecVT.getStoreSize() / TruncBytes); 48456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT); 48466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true); 48476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 48526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 48536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 4854de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineSIGN_EXTEND( 4855de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 4856de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Convert (sext (ashr (shl X, C1), C2)) to 4857de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as 4858de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // cheap as narrower ones. 485936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG = DCI.DAG; 4860de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue N0 = N->getOperand(0); 4861de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VT = N->getValueType(0); 4862de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) { 4863de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 4864de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Inner = N0.getOperand(0); 4865de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) { 4866de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) { 4867de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Extra = (VT.getSizeInBits() - 4868de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N0.getValueType().getSizeInBits()); 4869de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra; 4870de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NewSraAmt = SraAmt->getZExtValue() + Extra; 4871de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT ShiftVT = N0.getOperand(1).getValueType(); 4872de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT, 4873de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Inner.getOperand(0)); 4874de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext, 4875de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getConstant(NewShlAmt, SDLoc(Inner), 4876de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ShiftVT)); 4877de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, 4878de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT)); 48796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 48816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4882de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4883de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4884de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4885de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineMERGE( 4886de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 4887de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 4888de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Opcode = N->getOpcode(); 4889de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op0 = N->getOperand(0); 4890de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op1 = N->getOperand(1); 4891de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op0.getOpcode() == ISD::BITCAST) 4892de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op0 = Op0.getOperand(0); 4893de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op0.getOpcode() == SystemZISD::BYTE_MASK && 4894de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<ConstantSDNode>(Op0.getOperand(0))->getZExtValue() == 0) { 4895de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF 4896de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // for v4f32. 4897de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op1 == N->getOperand(0)) 4898de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op1; 4899de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (z_merge_? 0, X) -> (z_unpackl_? 0, X). 4900de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VT = Op1.getValueType(); 4901de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ElemBytes = VT.getVectorElementType().getStoreSize(); 4902de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ElemBytes <= 4) { 4903de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Opcode = (Opcode == SystemZISD::MERGE_HIGH ? 4904de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW); 4905de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT InVT = VT.changeVectorElementTypeToInteger(); 4906de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16), 4907de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SystemZ::VectorBytes / ElemBytes / 2); 4908de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT != InVT) { 4909de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1); 4910de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(Op1.getNode()); 49116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4912de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1); 4913de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(Op.getNode()); 4914de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); 49156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 49166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4917de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4918de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4919de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4920de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineSTORE( 4921de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 4922de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 4923de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *SN = cast<StoreSDNode>(N); 4924de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto &Op1 = N->getOperand(1); 4925de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT MemVT = SN->getMemoryVT(); 49266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better 49276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // for the extraction to be done on a vMiN value, so that we can use VSTE. 49286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // If X has wider elements then convert it to: 49296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z). 4930de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (MemVT.isInteger()) { 4931de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Value = 4932de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) { 4933de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(Value.getNode()); 4934de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4935de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Rewrite the store with the new form of stored value. 4936de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value, 4937de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SN->getBasePtr(), SN->getMemoryVT(), 4938de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SN->getMemOperand()); 49396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 49406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4941de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Combine STORE (BSWAP) into STRVH/STRV/STRVG 4942de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // See comment in combineBSWAP about volatile accesses. 4943de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!SN->isVolatile() && 4944de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op1.getOpcode() == ISD::BSWAP && 4945de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op1.getNode()->hasOneUse() && 4946de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (Op1.getValueType() == MVT::i16 || 4947de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op1.getValueType() == MVT::i32 || 4948de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op1.getValueType() == MVT::i64)) { 4949de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4950de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue BSwapOp = Op1.getOperand(0); 4951de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4952de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (BSwapOp.getValueType() == MVT::i16) 4953de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp); 4954de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4955de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Ops[] = { 4956de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(0), BSwapOp, N->getOperand(2), 4957de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getValueType(Op1.getValueType()) 4958de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }; 4959de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return 4961de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other), 4962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ops, MemVT, SN->getMemOperand()); 49636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4964de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4966de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4967de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT( 4968de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 4969de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Try to simplify a vector extraction. 4970de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 4971de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op0 = N->getOperand(0); 4972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VecVT = Op0.getValueType(); 4973de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0, 4974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IndexN->getZExtValue(), DCI, false); 49756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 4976de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4977de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4978de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4979de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineJOIN_DWORDS( 4980de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 4981de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 49826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (join_dwords X, X) == (replicate X) 4983de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N->getOperand(0) == N->getOperand(1)) 49846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0), 49856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar N->getOperand(0)); 4986de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4987de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4988de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4989de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineFP_ROUND( 4990de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 49916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (fround (extract_vector_elt X 0)) 49926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (fround (extract_vector_elt X 1)) -> 49936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (extract_vector_elt (VROUND X) 0) 49946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (extract_vector_elt (VROUND X) 1) 49956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // 49966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // This is a special case since the target doesn't really support v2f32s. 4997de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 4998de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Op0 = N->getOperand(0); 4999de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N->getValueType(0) == MVT::f32 && 5000de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op0.hasOneUse() && 5001de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 5002de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op0.getOperand(0).getValueType() == MVT::v2f64 && 5003de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op0.getOperand(1).getOpcode() == ISD::Constant && 5004de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) { 5005de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Vec = Op0.getOperand(0); 5006de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar for (auto *U : Vec->uses()) { 5007de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (U != Op0.getNode() && 5008de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar U->hasOneUse() && 5009de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar U->getOpcode() == ISD::EXTRACT_VECTOR_ELT && 5010de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar U->getOperand(0) == Vec && 5011de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar U->getOperand(1).getOpcode() == ISD::Constant && 5012de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 1) { 5013de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue OtherRound = SDValue(*U->use_begin(), 0); 5014de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (OtherRound.getOpcode() == ISD::FP_ROUND && 5015de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OtherRound.getOperand(0) == SDValue(U, 0) && 5016de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OtherRound.getValueType() == MVT::f32) { 5017de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N), 5018de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MVT::v4f32, Vec); 5019de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(VRound.getNode()); 5020de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Extract1 = 5021de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32, 5022de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar VRound, DAG.getConstant(2, SDLoc(U), MVT::i32)); 5023de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(Extract1.getNode()); 5024de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1); 5025de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Extract0 = 5026de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32, 5027de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32)); 5028de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Extract0; 502936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 503036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 503136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 503236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 503336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SDValue(); 503436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 503536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5036de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineBSWAP( 5037de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 5038de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 5039de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Combine BSWAP (LOAD) into LRVH/LRV/LRVG 5040de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // These loads are allowed to access memory multiple times, and so we must check 5041de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // that the loads are not volatile before performing the combine. 5042de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && 5043de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(0).hasOneUse() && 5044de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (N->getValueType(0) == MVT::i16 || N->getValueType(0) == MVT::i32 || 5045de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getValueType(0) == MVT::i64) && 5046de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar !cast<LoadSDNode>(N->getOperand(0))->isVolatile()) { 5047de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Load = N->getOperand(0); 5048de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LoadSDNode *LD = cast<LoadSDNode>(Load); 5049de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5050de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Create the byte-swapping load. 5051de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Ops[] = { 5052de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LD->getChain(), // Chain 5053de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LD->getBasePtr(), // Ptr 5054de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getValueType(N->getValueType(0)) // VT 5055de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }; 5056de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue BSLoad = 5057de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N), 5058de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getVTList(N->getValueType(0) == MVT::i64 ? 5059de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MVT::i64 : MVT::i32, MVT::Other), 5060de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ops, LD->getMemoryVT(), LD->getMemOperand()); 5061de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5062de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If this is an i16 load, insert the truncate. 5063de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue ResVal = BSLoad; 5064de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N->getValueType(0) == MVT::i16) 5065de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad); 5066de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5067de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // First, combine the bswap away. This makes the value produced by the 5068de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // load dead. 5069de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.CombineTo(N, ResVal); 5070de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5071de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Next, combine the load away, we give it a bogus result value but a real 5072de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // chain result. The result value is dead because the bswap is dead. 5073de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 5074de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5075de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Return N so it doesn't get rechecked! 5076de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(N, 0); 5077de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5078de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 5079de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 5080de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5081de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::combineSHIFTROT( 5082de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *N, DAGCombinerInfo &DCI) const { 5083de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5084de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 5085de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5086de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Shift/rotate instructions only use the last 6 bits of the second operand 5087de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // register. If the second operand is the result of an AND with an immediate 5088de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // value that has its last 6 bits set, we can safely remove the AND operation. 5089de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue N1 = N->getOperand(1); 5090de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N1.getOpcode() == ISD::AND) { 5091de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *AndMask = dyn_cast<ConstantSDNode>(N1.getOperand(1)); 5092de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The AND mask is constant 5094de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (AndMask) { 5095de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto AmtVal = AndMask->getZExtValue(); 5096de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5097de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Bottom 6 bits are set 5098de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if ((AmtVal & 0x3f) == 0x3f) { 5099de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue AndOp = N1->getOperand(0); 5100de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5101de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // This is the only use, so remove the node 5102de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N1.hasOneUse()) { 5103de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Combine the AND away 5104de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.CombineTo(N1.getNode(), AndOp); 5105de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5106de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Return N so it isn't rechecked 5107de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(N, 0); 5108de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5109de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The node will be reused, so create a new node for this one use 5110de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 5111de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Replace = DAG.getNode(N->getOpcode(), SDLoc(N), 5112de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getValueType(0), N->getOperand(0), 5113de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AndOp); 5114de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DCI.AddToWorklist(Replace.getNode()); 5115de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5116de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Replace; 5117de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5118de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5119de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5120de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5121de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5122de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 5123de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 5124de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5125de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, 5126de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAGCombinerInfo &DCI) const { 5127de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch(N->getOpcode()) { 5128de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar default: break; 5129de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI); 5130de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case SystemZISD::MERGE_HIGH: 5131de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI); 5132de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::STORE: return combineSTORE(N, DCI); 5133de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI); 5134de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI); 5135de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::FP_ROUND: return combineFP_ROUND(N, DCI); 5136de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::BSWAP: return combineBSWAP(N, DCI); 5137de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SHL: 5138de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SRA: 5139de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SRL: 5140de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::ROTL: return combineSHIFTROT(N, DCI); 5141de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 5142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5143de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 5144de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 5145de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 51461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 51471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Custom insertion 51481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand//===----------------------------------------------------------------------===// 51491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 51501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Create a new basic block after MBB. 51511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigandstatic MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 51521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = *MBB->getParent(); 51531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 515436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 51551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return NewMBB; 51561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 51571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 515847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford// Split MBB after MI and return the new block (the one that contains 515947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford// instructions after MI). 5160de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic MachineBasicBlock *splitBlockAfter(MachineBasicBlock::iterator MI, 516147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MachineBasicBlock *MBB) { 516247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 516347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford NewMBB->splice(NewMBB->begin(), MBB, 516436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 516547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 516647e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford return NewMBB; 516747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford} 516847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford 5169842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford// Split MBB before MI and return the new block (the one that contains MI). 5170de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic MachineBasicBlock *splitBlockBefore(MachineBasicBlock::iterator MI, 5171842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *MBB) { 51721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 5173842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 51741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 51751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return NewMBB; 51761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 51771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5178842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford// Force base value Base into a register before MI. Return the register. 5179de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic unsigned forceReg(MachineInstr &MI, MachineOperand &Base, 5180842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford const SystemZInstrInfo *TII) { 5181842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford if (Base.isReg()) 5182842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford return Base.getReg(); 5183842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5184de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineBasicBlock *MBB = MI.getParent(); 5185842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineFunction &MF = *MBB->getParent(); 5186842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineRegisterInfo &MRI = MF.getRegInfo(); 5187842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5188842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 5189de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg) 5190de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addOperand(Base) 5191de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addImm(0) 5192de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addReg(0); 5193842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford return Reg; 5194842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford} 5195842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 51961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 51971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandMachineBasicBlock * 5198de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSystemZTargetLowering::emitSelect(MachineInstr &MI, 51991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *MBB) const { 5200ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SystemZInstrInfo *TII = 5201ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 52021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5203de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned DestReg = MI.getOperand(0).getReg(); 5204de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned TrueReg = MI.getOperand(1).getReg(); 5205de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned FalseReg = MI.getOperand(2).getReg(); 5206de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned CCValid = MI.getOperand(3).getImm(); 5207de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned CCMask = MI.getOperand(4).getImm(); 5208de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 52091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 52101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *StartMBB = MBB; 5211842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 52121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 52131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 52141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // StartMBB: 5215d50bcb2162a529534da42748ab4a418bfc9aaf06Richard Sandiford // BRC CCMask, JoinMBB 52161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fallthrough to FalseMBB 52171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = StartMBB; 52186824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 52196824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 52201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(JoinMBB); 52211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(FalseMBB); 52221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 52231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // FalseMBB: 52241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fallthrough to JoinMBB 52251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = FalseMBB; 52261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(JoinMBB); 52271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 52281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // JoinMBB: 52291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 52301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ... 52311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = JoinMBB; 5232842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg) 52331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(TrueReg).addMBB(StartMBB) 52341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(FalseReg).addMBB(FalseMBB); 52351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5236de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 52371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return JoinMBB; 52381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 52391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5240722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 5241722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford// StoreOpcode is the store to use and Invert says whether the store should 5242b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford// happen when the condition is false rather than true. If a STORE ON 5243b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 5244de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI, 5245de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineBasicBlock *MBB, 5246de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned StoreOpcode, 5247de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned STOCOpcode, 5248de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool Invert) const { 5249ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SystemZInstrInfo *TII = 5250ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 5251722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5252de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned SrcReg = MI.getOperand(0).getReg(); 5253de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand Base = MI.getOperand(1); 5254de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int64_t Disp = MI.getOperand(2).getImm(); 5255de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned IndexReg = MI.getOperand(3).getReg(); 5256de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned CCValid = MI.getOperand(4).getImm(); 5257de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned CCMask = MI.getOperand(5).getImm(); 5258de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 5259722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5260722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 5261722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5262b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford // Use STOCOpcode if possible. We could use different store patterns in 5263b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford // order to avoid matching the index register, but the performance trade-offs 5264b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford // might be more complicated in that case. 5265c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) { 5266b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford if (Invert) 52676824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford CCMask ^= CCValid; 5268b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 52698f0ad5ae8f2699f6ab13a229941a0b192273cae8Richard Sandiford .addReg(SrcReg).addOperand(Base).addImm(Disp) 52708f0ad5ae8f2699f6ab13a229941a0b192273cae8Richard Sandiford .addImm(CCValid).addImm(CCMask); 5271de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 5272b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return MBB; 5273b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford } 5274b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford 5275722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // Get the condition needed to branch around the store. 5276722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford if (!Invert) 52776824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford CCMask ^= CCValid; 5278722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5279722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MachineBasicBlock *StartMBB = MBB; 5280842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 5281722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 5282722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5283722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // StartMBB: 5284722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // BRC CCMask, JoinMBB 5285722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // # fallthrough to FalseMBB 5286722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MBB = StartMBB; 52876824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 52886824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 5289722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MBB->addSuccessor(JoinMBB); 5290722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MBB->addSuccessor(FalseMBB); 5291722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5292722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // FalseMBB: 5293722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // store %SrcReg, %Disp(%Index,%Base) 5294722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford // # fallthrough to JoinMBB 5295722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MBB = FalseMBB; 5296722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford BuildMI(MBB, DL, TII->get(StoreOpcode)) 5297722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 5298722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford MBB->addSuccessor(JoinMBB); 5299722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 5300de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 5301722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford return JoinMBB; 5302722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford} 5303722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 53041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 53051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 53061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 53071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// BitSize is the width of the field in bits, or 0 if this is a partword 53081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 53091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// is one of the operands. Invert says whether the field should be 53101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// inverted after performing BinOpcode (e.g. for NAND). 5311de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( 5312de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode, 5313de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned BitSize, bool Invert) const { 53141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = *MBB->getParent(); 5315c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5316ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 53171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineRegisterInfo &MRI = MF.getRegInfo(); 53181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand bool IsSubWord = (BitSize < 32); 53191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Extract the operands. Base can be a register or a frame index. 53211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Src2 can be a register or immediate. 5322de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Dest = MI.getOperand(0).getReg(); 5323de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand Base = earlyUseOperand(MI.getOperand(1)); 5324de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int64_t Disp = MI.getOperand(2).getImm(); 5325de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand Src2 = earlyUseOperand(MI.getOperand(3)); 5326de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned BitShift = (IsSubWord ? MI.getOperand(4).getReg() : 0); 5327de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : 0); 5328de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 53291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 5330de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BitSize = MI.getOperand(6).getImm(); 53311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Subword operations use 32-bit registers. 53331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const TargetRegisterClass *RC = (BitSize <= 32 ? 53341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand &SystemZ::GR32BitRegClass : 53351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand &SystemZ::GR64BitRegClass); 53361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 53371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 53381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the right opcodes for the displacement. 53401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 53411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 53421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(LOpcode && CSOpcode && "Displacement out of range"); 53431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Create virtual registers for temporary results. 53451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OrigVal = MRI.createVirtualRegister(RC); 53461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OldVal = MRI.createVirtualRegister(RC); 53471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NewVal = (BinOpcode || IsSubWord ? 53481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MRI.createVirtualRegister(RC) : Src2.getReg()); 53491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 53501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 53511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Insert a basic block for the main loop. 53531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *StartMBB = MBB; 5354842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 53551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 53561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // StartMBB: 53581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ... 53591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OrigVal = L Disp(%Base) 53601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to LoopMMB 53611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = StartMBB; 53621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 53631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addOperand(Base).addImm(Disp).addReg(0); 53641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 53651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 53661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // LoopMBB: 53671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 53681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 53691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RotatedNewVal = OP %RotatedOldVal, %Src2 53701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 53711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %Dest = CS %OldVal, %NewVal, Disp(%Base) 53721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // JNE LoopMBB 53731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to DoneMMB 53741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = LoopMBB; 53751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 53761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OrigVal).addMBB(StartMBB) 53771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(Dest).addMBB(LoopMBB); 53781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 53791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 53801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(BitShift).addImm(0); 53811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (Invert) { 53821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Perform the operation normally and then invert every bit of the field. 53831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Tmp = MRI.createVirtualRegister(RC); 53841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 53851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedOldVal).addOperand(Src2); 538637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (BitSize <= 32) 53871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // XILF with the upper BitSize bits set. 5388259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 538937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines .addReg(Tmp).addImm(-1U << (32 - BitSize)); 53901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else { 53911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Use LCGR and add -1 to the result, which is more compact than 53921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // an XILF, XILH pair. 53931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Tmp2 = MRI.createVirtualRegister(RC); 53941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 53951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 53961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(Tmp2).addImm(-1); 53971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 53981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } else if (BinOpcode) 53991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // A simply binary operation. 54001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 54011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedOldVal).addOperand(Src2); 54021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand else if (IsSubWord) 54031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Use RISBG to rotate Src2 into position and use it to replace the 54041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // field in RotatedOldVal. 54051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 54061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedOldVal).addReg(Src2.getReg()) 54071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 54081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 54091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 54101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 54111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 54121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 54136824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 54146824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 54151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 54161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(DoneMBB); 54171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5418de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 54191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DoneMBB; 54201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 54211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Implement EmitInstrWithCustomInserter for pseudo 54231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 54241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// instruction that should be used to compare the current field with the 54251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// minimum or maximum value. KeepOldMask is the BRC condition-code mask 54261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// for when the current field should be kept. BitSize is the width of 54271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 5428de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( 5429de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode, 5430de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned KeepOldMask, unsigned BitSize) const { 54311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = *MBB->getParent(); 5432c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5433ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 54341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineRegisterInfo &MRI = MF.getRegInfo(); 54351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand bool IsSubWord = (BitSize < 32); 54361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Extract the operands. Base can be a register or a frame index. 5438de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Dest = MI.getOperand(0).getReg(); 5439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand Base = earlyUseOperand(MI.getOperand(1)); 5440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int64_t Disp = MI.getOperand(2).getImm(); 5441de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Src2 = MI.getOperand(3).getReg(); 5442de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned BitShift = (IsSubWord ? MI.getOperand(4).getReg() : 0); 5443de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : 0); 5444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 54451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 5446de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BitSize = MI.getOperand(6).getImm(); 54471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Subword operations use 32-bit registers. 54491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const TargetRegisterClass *RC = (BitSize <= 32 ? 54501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand &SystemZ::GR32BitRegClass : 54511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand &SystemZ::GR64BitRegClass); 54521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 54531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 54541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the right opcodes for the displacement. 54561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 54571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 54581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(LOpcode && CSOpcode && "Displacement out of range"); 54591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Create virtual registers for temporary results. 54611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OrigVal = MRI.createVirtualRegister(RC); 54621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OldVal = MRI.createVirtualRegister(RC); 54631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NewVal = MRI.createVirtualRegister(RC); 54641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 54651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 54661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 54671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Insert 3 basic blocks for the loop. 54691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *StartMBB = MBB; 5470842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 54711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 54721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 54731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 54741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // StartMBB: 54761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ... 54771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OrigVal = L Disp(%Base) 54781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to LoopMMB 54791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = StartMBB; 54801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 54811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addOperand(Base).addImm(Disp).addReg(0); 54821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 54831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 54841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // LoopMBB: 54851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 54861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 54871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // CompareOpcode %RotatedOldVal, %Src2 548844b486ed78c60b50aa14d4eed92ee828d4d44293Richard Sandiford // BRC KeepOldMask, UpdateMBB 54891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = LoopMBB; 54901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 54911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OrigVal).addMBB(StartMBB) 54921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(Dest).addMBB(UpdateMBB); 54931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 54941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 54951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(BitShift).addImm(0); 54960416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford BuildMI(MBB, DL, TII->get(CompareOpcode)) 54970416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford .addReg(RotatedOldVal).addReg(Src2); 54980416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 54996824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 55001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(UpdateMBB); 55011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(UseAltMBB); 55021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // UseAltMBB: 55041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 55051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to UpdateMMB 55061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = UseAltMBB; 55071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 55081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 55091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedOldVal).addReg(Src2) 55101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addImm(32).addImm(31 + BitSize).addImm(0); 55111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(UpdateMBB); 55121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // UpdateMBB: 55141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 55151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // [ %RotatedAltVal, UseAltMBB ] 55161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 55171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %Dest = CS %OldVal, %NewVal, Disp(%Base) 55181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // JNE LoopMBB 55191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to DoneMMB 55201d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = UpdateMBB; 55211d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 55221d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedOldVal).addMBB(LoopMBB) 55231d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedAltVal).addMBB(UseAltMBB); 55241d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (IsSubWord) 55251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 55261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 55271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 55281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 55296824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 55306824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 55311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 55321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(DoneMBB); 55331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5534de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 55351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DoneMBB; 55361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 55371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 55391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// instruction MI. 55401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich WeigandMachineBasicBlock * 5541de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, 55421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *MBB) const { 5543de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 55441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = *MBB->getParent(); 5545c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5546ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 55471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineRegisterInfo &MRI = MF.getRegInfo(); 55481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Extract the operands. Base can be a register or a frame index. 5550de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Dest = MI.getOperand(0).getReg(); 5551de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand Base = earlyUseOperand(MI.getOperand(1)); 5552de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int64_t Disp = MI.getOperand(2).getImm(); 5553de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned OrigCmpVal = MI.getOperand(3).getReg(); 5554de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned OrigSwapVal = MI.getOperand(4).getReg(); 5555de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned BitShift = MI.getOperand(5).getReg(); 5556de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NegBitShift = MI.getOperand(6).getReg(); 5557de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int64_t BitSize = MI.getOperand(7).getImm(); 5558de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 55591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 55611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Get the right opcodes for the displacement. 55631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 55641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 55651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand assert(LOpcode && CSOpcode && "Displacement out of range"); 55661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Create virtual registers for temporary results. 55681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OrigOldVal = MRI.createVirtualRegister(RC); 55691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned OldVal = MRI.createVirtualRegister(RC); 55701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned CmpVal = MRI.createVirtualRegister(RC); 55711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned SwapVal = MRI.createVirtualRegister(RC); 55721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned StoreVal = MRI.createVirtualRegister(RC); 55731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RetryOldVal = MRI.createVirtualRegister(RC); 55741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 55751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 55761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // Insert 2 basic blocks for the loop. 55781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *StartMBB = MBB; 5579842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 55801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 55811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 55821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // StartMBB: 55841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ... 55851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OrigOldVal = L Disp(%Base) 55861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to LoopMMB 55871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = StartMBB; 55881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 55891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addOperand(Base).addImm(Disp).addReg(0); 55901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 55911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 55921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // LoopMBB: 55931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 55941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 55951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 55961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %Dest = RLL %OldVal, BitSize(%BitShift) 55971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ^^ The low BitSize bits contain the field 55981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // of interest. 55991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 56001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ^^ Replace the upper 32-BitSize bits of the 56011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // comparison value with those that we loaded, 56021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // so that we can use a full word comparison. 56030416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford // CR %Dest, %RetryCmpVal 56040416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford // JNE DoneMBB 56051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # Fall through to SetMBB 56061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = LoopMBB; 56071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 56081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OrigOldVal).addMBB(StartMBB) 56091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RetryOldVal).addMBB(SetMBB); 56101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 56111d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OrigCmpVal).addMBB(StartMBB) 56121d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RetryCmpVal).addMBB(SetMBB); 56131d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 56141d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OrigSwapVal).addMBB(StartMBB) 56151d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RetrySwapVal).addMBB(SetMBB); 56161d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 56171d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(BitShift).addImm(BitSize); 56181d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 56191d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 56200416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::CR)) 56210416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford .addReg(Dest).addReg(RetryCmpVal); 56220416e3c599c22dc656a1115ac983116ad0b2d9daRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 56236824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_ICMP) 56246824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 56251d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(DoneMBB); 56261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(SetMBB); 56271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 56281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // SetMBB: 56291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 56301d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ^^ Replace the upper 32-BitSize bits of the new 56311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // value with those that we loaded. 56321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 56331d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // ^^ Rotate the new field to its proper position. 56341d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 56351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // JNE LoopMBB 56361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand // # fall through to ExitMMB 56371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB = SetMBB; 56381d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 56391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 56401d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 56411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 56421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 56431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 56446824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 56456824f127f90197b26af93cf5d6c13b7941567e54Richard Sandiford .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 56461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(LoopMBB); 56471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MBB->addSuccessor(DoneMBB); 56481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5649de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 56501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return DoneMBB; 56511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 56521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 56531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 56541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand// if the high register of the GR128 value must be cleared or false if 5655745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford// it's "don't care". SubReg is subreg_l32 when extending a GR32 5656745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford// and subreg_l64 when extending a GR64. 5657de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI, 5658de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineBasicBlock *MBB, 5659de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool ClearEven, 5660de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned SubReg) const { 56611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineFunction &MF = *MBB->getParent(); 5662c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5663ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 56641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand MachineRegisterInfo &MRI = MF.getRegInfo(); 5665de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 56661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5667de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Dest = MI.getOperand(0).getReg(); 5668de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Src = MI.getOperand(1).getReg(); 56691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 56701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 56711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 56721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand if (ClearEven) { 56731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 56741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 56751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 56761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 56771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addImm(0); 56781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 5679745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64); 56801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand In128 = NewIn128; 56811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 56821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 56831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand .addReg(In128).addReg(Src).addImm(SubReg); 56841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5685de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 56861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return MBB; 56871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 56881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 5689de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( 5690de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const { 5691842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineFunction &MF = *MBB->getParent(); 5692c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5693ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 5694842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineRegisterInfo &MRI = MF.getRegInfo(); 5695de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 5696dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford 5697de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand DestBase = earlyUseOperand(MI.getOperand(0)); 5698de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t DestDisp = MI.getOperand(1).getImm(); 5699de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2)); 5700de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t SrcDisp = MI.getOperand(3).getImm(); 5701de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t Length = MI.getOperand(4).getImm(); 5702dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford 570347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // When generating more than one CLC, all but the last will need to 570447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // branch to the end when a difference is found. 570547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ? 5706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines splitBlockAfter(MI, MBB) : nullptr); 570747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford 5708842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // Check for the loop form, in which operand 5 is the trip count. 5709de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (MI.getNumExplicitOperands() > 5) { 5710842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); 5711842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5712de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t StartCountReg = MI.getOperand(5).getReg(); 5713842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t StartSrcReg = forceReg(MI, SrcBase, TII); 5714842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t StartDestReg = (HaveSingleBase ? StartSrcReg : 5715842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford forceReg(MI, DestBase, TII)); 5716842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5717842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass; 5718842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t ThisSrcReg = MRI.createVirtualRegister(RC); 5719842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t ThisDestReg = (HaveSingleBase ? ThisSrcReg : 5720842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MRI.createVirtualRegister(RC)); 5721842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t NextSrcReg = MRI.createVirtualRegister(RC); 5722842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t NextDestReg = (HaveSingleBase ? NextSrcReg : 5723842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MRI.createVirtualRegister(RC)); 5724842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5725842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford RC = &SystemZ::GR64BitRegClass; 5726842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t ThisCountReg = MRI.createVirtualRegister(RC); 5727842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t NextCountReg = MRI.createVirtualRegister(RC); 5728842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5729842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *StartMBB = MBB; 5730842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 5731842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 573247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MachineBasicBlock *NextMBB = (EndMBB ? emitBlockAfter(LoopMBB) : LoopMBB); 5733842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5734842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // StartMBB: 5735842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // # fall through to LoopMMB 5736842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MBB->addSuccessor(LoopMBB); 5737842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5738842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // LoopMBB: 5739842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // %ThisDestReg = phi [ %StartDestReg, StartMBB ], 574047e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // [ %NextDestReg, NextMBB ] 5741842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ], 574247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // [ %NextSrcReg, NextMBB ] 5743842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // %ThisCountReg = phi [ %StartCountReg, StartMBB ], 574447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // [ %NextCountReg, NextMBB ] 574547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // ( PFD 2, 768+DestDisp(%ThisDestReg) ) 5746842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg) 574747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // ( JLH EndMBB ) 5748842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // 574947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // The prefetch is used only for MVC. The JLH is used only for CLC. 5750842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MBB = LoopMBB; 5751842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5752842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) 5753842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(StartDestReg).addMBB(StartMBB) 575447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addReg(NextDestReg).addMBB(NextMBB); 5755842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford if (!HaveSingleBase) 5756842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) 5757842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(StartSrcReg).addMBB(StartMBB) 575847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addReg(NextSrcReg).addMBB(NextMBB); 5759842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) 5760842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(StartCountReg).addMBB(StartMBB) 576147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addReg(NextCountReg).addMBB(NextMBB); 576247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford if (Opcode == SystemZ::MVC) 576347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PFD)) 576447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addImm(SystemZ::PFD_WRITE) 576547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); 5766842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(Opcode)) 5767842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(ThisDestReg).addImm(DestDisp).addImm(256) 5768842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(ThisSrcReg).addImm(SrcDisp); 576947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford if (EndMBB) { 577047e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 577147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 577247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addMBB(EndMBB); 577347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addSuccessor(EndMBB); 577447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addSuccessor(NextMBB); 577547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford } 577647e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford 577747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // NextMBB: 577847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // %NextDestReg = LA 256(%ThisDestReg) 577947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // %NextSrcReg = LA 256(%ThisSrcReg) 578047e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // %NextCountReg = AGHI %ThisCountReg, -1 578147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // CGHI %NextCountReg, 0 578247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // JLH LoopMBB 578347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // # fall through to DoneMMB 578447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // 578547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // The AGHI, CGHI and JLH should be converted to BRCTG by later passes. 578647e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB = NextMBB; 578747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford 5788842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) 5789842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(ThisDestReg).addImm(256).addReg(0); 5790842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford if (!HaveSingleBase) 5791842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) 5792842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(ThisSrcReg).addImm(256).addReg(0); 5793842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) 5794842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(ThisCountReg).addImm(-1); 5795842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) 5796842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addReg(NextCountReg).addImm(0); 5797842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 5798842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 5799842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addMBB(LoopMBB); 5800842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MBB->addSuccessor(LoopMBB); 5801842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MBB->addSuccessor(DoneMBB); 5802842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford 5803842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford DestBase = MachineOperand::CreateReg(NextDestReg, false); 5804842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford SrcBase = MachineOperand::CreateReg(NextSrcReg, false); 5805842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford Length &= 255; 5806842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MBB = DoneMBB; 5807842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford } 5808842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // Handle any remaining bytes with straight-line code. 5809842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford while (Length > 0) { 5810842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford uint64_t ThisLength = std::min(Length, uint64_t(256)); 5811842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // The previous iteration might have created out-of-range displacements. 5812842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford // Apply them using LAY if so. 5813842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford if (!isUInt<12>(DestDisp)) { 5814842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 5815de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg) 5816de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addOperand(DestBase) 5817de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addImm(DestDisp) 5818de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addReg(0); 5819842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford DestBase = MachineOperand::CreateReg(Reg, false); 5820842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford DestDisp = 0; 5821842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford } 5822842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford if (!isUInt<12>(SrcDisp)) { 5823842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 5824de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg) 5825de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addOperand(SrcBase) 5826de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addImm(SrcDisp) 5827de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .addReg(0); 5828842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford SrcBase = MachineOperand::CreateReg(Reg, false); 5829842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford SrcDisp = 0; 5830842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford } 5831842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford BuildMI(*MBB, MI, DL, TII->get(Opcode)) 5832842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addOperand(DestBase).addImm(DestDisp).addImm(ThisLength) 5833842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford .addOperand(SrcBase).addImm(SrcDisp); 5834842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford DestDisp += ThisLength; 5835842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford SrcDisp += ThisLength; 5836842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford Length -= ThisLength; 583747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // If there's another CLC to go, branch to the end if a difference 583847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford // was found. 583947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford if (EndMBB && Length > 0) { 584047e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MachineBasicBlock *NextMBB = splitBlockBefore(MI, MBB); 584147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 584247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 584347e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford .addMBB(EndMBB); 584447e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addSuccessor(EndMBB); 584547e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addSuccessor(NextMBB); 584647e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB = NextMBB; 584747e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford } 584847e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford } 584947e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford if (EndMBB) { 585047e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addSuccessor(EndMBB); 585147e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB = EndMBB; 585247e70960945ecb33a361987a9745e3dc80a1c78cRichard Sandiford MBB->addLiveIn(SystemZ::CC); 5853842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford } 5854dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford 5855de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 5856dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford return MBB; 5857dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford} 5858dff0009d0ced62b92cb5900bc2203ec40142ba15Richard Sandiford 5859e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford// Decompose string pseudo-instruction MI into a loop that continually performs 5860e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford// Opcode until CC != 3. 5861de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitStringWrapper( 5862de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const { 5863e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MachineFunction &MF = *MBB->getParent(); 5864c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const SystemZInstrInfo *TII = 5865ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 5866e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MachineRegisterInfo &MRI = MF.getRegInfo(); 5867de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 5868e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5869de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t End1Reg = MI.getOperand(0).getReg(); 5870de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t Start1Reg = MI.getOperand(1).getReg(); 5871de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t Start2Reg = MI.getOperand(2).getReg(); 5872de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t CharReg = MI.getOperand(3).getReg(); 5873e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5874e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 5875e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford uint64_t This1Reg = MRI.createVirtualRegister(RC); 5876e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford uint64_t This2Reg = MRI.createVirtualRegister(RC); 5877e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford uint64_t End2Reg = MRI.createVirtualRegister(RC); 5878e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5879e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MachineBasicBlock *StartMBB = MBB; 5880842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 5881e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 5882e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5883e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // StartMBB: 5884e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // # fall through to LoopMMB 5885e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MBB->addSuccessor(LoopMBB); 5886e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5887e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // LoopMBB: 5888e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 5889e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 5890514756983e9ba3684a89ed583bf5a98ffb20c203Richard Sandiford // R0L = %CharReg 5891514756983e9ba3684a89ed583bf5a98ffb20c203Richard Sandiford // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L 5892e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // JO LoopMBB 5893e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford // # fall through to DoneMMB 58948c20158fb0e1e5d747077f065eb0170c5af1fbfaRichard Sandiford // 5895514756983e9ba3684a89ed583bf5a98ffb20c203Richard Sandiford // The load of R0L can be hoisted by post-RA LICM. 5896e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MBB = LoopMBB; 5897e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5898e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 5899e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(Start1Reg).addMBB(StartMBB) 5900e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(End1Reg).addMBB(LoopMBB); 5901e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 5902e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(Start2Reg).addMBB(StartMBB) 5903e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(End2Reg).addMBB(LoopMBB); 5904514756983e9ba3684a89ed583bf5a98ffb20c203Richard Sandiford BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); 5905e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford BuildMI(MBB, DL, TII->get(Opcode)) 5906e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 5907e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addReg(This1Reg).addReg(This2Reg); 5908e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 5909e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 5910e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MBB->addSuccessor(LoopMBB); 5911e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford MBB->addSuccessor(DoneMBB); 5912e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5913e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford DoneMBB->addLiveIn(SystemZ::CC); 5914e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 5915de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 5916e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford return DoneMBB; 5917e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford} 5918e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford 59190c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar// Update TBEGIN instruction with final opcode and register clobbers. 5920de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitTransactionBegin( 5921de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode, 5922de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool NoFloat) const { 59230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar MachineFunction &MF = *MBB->getParent(); 59240c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); 59250c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar const SystemZInstrInfo *TII = Subtarget.getInstrInfo(); 59260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 59270c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Update opcode. 5928de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.setDesc(TII->get(Opcode)); 59290c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 59300c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // We cannot handle a TBEGIN that clobbers the stack or frame pointer. 59310c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Make sure to add the corresponding GRSM bits if they are missing. 5932de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t Control = MI.getOperand(2).getImm(); 59330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar static const unsigned GPRControlBit[16] = { 59340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000, 59350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100 59360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar }; 59370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Control |= GPRControlBit[15]; 59380c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (TFI->hasFP(MF)) 59390c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Control |= GPRControlBit[11]; 5940de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.getOperand(2).setImm(Control); 59410c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 59420c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Add GPR clobbers. 59430c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar for (int I = 0; I < 16; I++) { 59440c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if ((Control & GPRControlBit[I]) == 0) { 59450c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Reg = SystemZMC::GR64Regs[I]; 5946de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.addOperand(MachineOperand::CreateReg(Reg, true, true)); 59470c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 59480c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 59490c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 59506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Add FPR/VR clobbers. 59510c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (!NoFloat && (Control & 4) != 0) { 59526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget.hasVector()) { 59536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (int I = 0; I < 32; I++) { 59546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Reg = SystemZMC::VR128Regs[I]; 5955de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.addOperand(MachineOperand::CreateReg(Reg, true, true)); 59566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 59576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 59586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (int I = 0; I < 16; I++) { 59596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Reg = SystemZMC::FP64Regs[I]; 5960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.addOperand(MachineOperand::CreateReg(Reg, true, true)); 59616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 59620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 59630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 59640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 59650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return MBB; 59660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 59670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 5968de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0( 5969de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const { 5970f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineFunction &MF = *MBB->getParent(); 5971f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineRegisterInfo *MRI = &MF.getRegInfo(); 5972f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const SystemZInstrInfo *TII = 5973f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); 5974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 5975f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 5976de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned SrcReg = MI.getOperand(0).getReg(); 5977f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 5978f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Create new virtual register of the same class as source. 5979f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetRegisterClass *RC = MRI->getRegClass(SrcReg); 5980f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned DstReg = MRI->createVirtualRegister(RC); 5981f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 5982f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Replace pseudo with a normal load-and-test that models the def as 5983f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // well. 5984f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg) 5985f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar .addReg(SrcReg); 5986de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 5987f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 5988f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return MBB; 5989f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 5990f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 5991de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter( 5992de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *MBB) const { 5993de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (MI.getOpcode()) { 599447086570973e82fe5ea8ace9637ae73c2469e1daRichard Sandiford case SystemZ::Select32Mux: 59951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::Select32: 59961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::SelectF32: 59971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::Select64: 59981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::SelectF64: 59991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::SelectF128: 60001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitSelect(MI, MBB); 60011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 600200f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford case SystemZ::CondStore8Mux: 600300f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false); 600400f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford case SystemZ::CondStore8MuxInv: 600500f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true); 600600f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford case SystemZ::CondStore16Mux: 600700f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false); 600800f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford case SystemZ::CondStore16MuxInv: 600900f5335ea0b62f0921d215a4d04e2fe5f33771ceRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true); 6010722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore8: 6011b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 6012722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore8Inv: 6013b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 6014722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore16: 6015b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 6016722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore16Inv: 6017b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 6018722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore32: 6019b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 6020722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore32Inv: 6021b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 6022722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore64: 6023b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 6024722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStore64Inv: 6025b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 6026722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStoreF32: 6027b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 6028722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStoreF32Inv: 6029b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 6030722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStoreF64: 6031b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 6032722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford case SystemZ::CondStoreF64Inv: 6033b284e1bf08d24deb20b7deab71fce6f3034cc89aRichard Sandiford return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 6034722e9e6d0a5b67d136be40bc015abc5b0b32f97bRichard Sandiford 60351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::AEXT128_64: 6036745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford return emitExt128(MI, MBB, false, SystemZ::subreg_l64); 60371d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ZEXT128_32: 6038745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford return emitExt128(MI, MBB, true, SystemZ::subreg_l32); 60391d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ZEXT128_64: 6040745ca1eed7dc0a056b066f16aea750ce6fa8a530Richard Sandiford return emitExt128(MI, MBB, true, SystemZ::subreg_l64); 60411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 60421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_SWAPW: 60431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, 0, 0); 60441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_SWAP_32: 60451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, 0, 32); 60461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_SWAP_64: 60471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, 0, 64); 60481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 60491d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_AR: 60501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 60511d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_AFI: 60521d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 60531d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AR: 60541d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 60551d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AHI: 60561d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 60571d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AFI: 60581d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 60591d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AGR: 60601d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 60611d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AGHI: 60621d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 60631d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_AGFI: 60641d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 60651d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 60661d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_SR: 60671d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 60681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_SR: 60691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 60701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_SGR: 60711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 60721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 60731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_NR: 60741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 60751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_NILH: 6076259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); 60771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NR: 60781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 60791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NILL: 6080259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); 60811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NILH: 6082259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); 6083259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILF: 6084259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); 6085259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NGR: 6086259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 6087259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILL64: 6088259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); 6089259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILH64: 6090259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); 6091b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHL64: 6092b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); 6093b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHH64: 6094b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); 6095259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILF64: 6096259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); 6097b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHF64: 6098b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); 60991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_OR: 61011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 61021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_OILH: 6103259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); 61041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_OR: 61051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 61061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_OILL: 6107259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); 61081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_OILH: 6109259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); 6110259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_OILF: 6111259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); 6112259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_OGR: 6113259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 6114259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_OILL64: 6115259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); 6116259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_OILH64: 6117259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); 61181c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford case SystemZ::ATOMIC_LOAD_OIHL64: 61191c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); 61201c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford case SystemZ::ATOMIC_LOAD_OIHH64: 61211c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); 6122259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_OILF64: 6123259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); 61241c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford case SystemZ::ATOMIC_LOAD_OIHF64: 61251c831f7f1f7e869516f5a70c1e6dd8896bef76f5Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); 61261d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61271d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_XR: 61281d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 61291d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_XILF: 6130259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); 61311d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_XR: 61321d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 6133259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_XILF: 6134259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); 61351d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_XGR: 61361d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 6137259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_XILF64: 6138259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); 61391ff62e182e648c72e6fce4f9d7911f2edfd914d2Richard Sandiford case SystemZ::ATOMIC_LOAD_XIHF64: 61401ff62e182e648c72e6fce4f9d7911f2edfd914d2Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); 61411d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61421d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_NRi: 61431d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 61441d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_NILHi: 6145259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); 61461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NRi: 61471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 61481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NILLi: 6149259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); 61501d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_NILHi: 6151259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); 6152259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILFi: 6153259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); 6154259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NGRi: 6155259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 6156259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILL64i: 6157259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); 6158259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILH64i: 6159259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); 6160b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHL64i: 6161b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); 6162b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHH64i: 6163b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); 6164259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford case SystemZ::ATOMIC_LOAD_NILF64i: 6165259a6006e89576704e52e7392ef2bfd83f277ce3Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); 6166b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford case SystemZ::ATOMIC_LOAD_NIHF64i: 6167b9dcca8265e9da01119c47e65f114c3adc972ba6Richard Sandiford return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); 61681d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61691d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_MIN: 61701d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 61711d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 0); 61721d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_MIN_32: 61731d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 61741d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 32); 61751d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_MIN_64: 61761d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 61771d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 64); 61781d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61791d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_MAX: 61801d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 61811d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 0); 61821d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_MAX_32: 61831d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 61841d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 32); 61851d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_MAX_64: 61861d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 61871d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 64); 61881d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61891d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_UMIN: 61901d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 61911d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 0); 61921d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_UMIN_32: 61931d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 61941d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 32); 61951d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_UMIN_64: 61961d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 61971d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_LE, 64); 61981d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 61991d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOADW_UMAX: 62001d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 62011d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 0); 62021d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_UMAX_32: 62031d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 62041d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 32); 62051d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_LOAD_UMAX_64: 62061d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 62071d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand SystemZ::CCMASK_CMP_GE, 64); 62081d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand 62091d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand case SystemZ::ATOMIC_CMP_SWAPW: 62101d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand return emitAtomicCmpSwapW(MI, MBB); 6211842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford case SystemZ::MVCSequence: 6212842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford case SystemZ::MVCLoop: 6213ac168b8bc8773a083a10902f64e4ae57a925aee4Richard Sandiford return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 621416277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::NCSequence: 621516277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::NCLoop: 621616277c4698f36a756c540fae326874774156aaedRichard Sandiford return emitMemMemWrapper(MI, MBB, SystemZ::NC); 621716277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::OCSequence: 621816277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::OCLoop: 621916277c4698f36a756c540fae326874774156aaedRichard Sandiford return emitMemMemWrapper(MI, MBB, SystemZ::OC); 622016277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::XCSequence: 622116277c4698f36a756c540fae326874774156aaedRichard Sandiford case SystemZ::XCLoop: 622216277c4698f36a756c540fae326874774156aaedRichard Sandiford return emitMemMemWrapper(MI, MBB, SystemZ::XC); 6223842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford case SystemZ::CLCSequence: 6224842a1be06c53757e7498c9894abc1431b633a92fRichard Sandiford case SystemZ::CLCLoop: 6225ac168b8bc8773a083a10902f64e4ae57a925aee4Richard Sandiford return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 6226e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford case SystemZ::CLSTLoop: 6227e1b2af731e2a45344a7c502232f66c55cd746da0Richard Sandiford return emitStringWrapper(MI, MBB, SystemZ::CLST); 62284fc7355a21e1fa838406e15459aaf54a58fcf909Richard Sandiford case SystemZ::MVSTLoop: 62294fc7355a21e1fa838406e15459aaf54a58fcf909Richard Sandiford return emitStringWrapper(MI, MBB, SystemZ::MVST); 623019262ee0725a09b7c621a3d2eb66ba1513ae932aRichard Sandiford case SystemZ::SRSTLoop: 623119262ee0725a09b7c621a3d2eb66ba1513ae932aRichard Sandiford return emitStringWrapper(MI, MBB, SystemZ::SRST); 62320c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case SystemZ::TBEGIN: 62330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, false); 62340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case SystemZ::TBEGIN_nofloat: 62350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, true); 62360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case SystemZ::TBEGINC: 62370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return emitTransactionBegin(MI, MBB, SystemZ::TBEGINC, true); 6238f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case SystemZ::LTEBRCompare_VecPseudo: 6239f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTEBR); 6240f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case SystemZ::LTDBRCompare_VecPseudo: 6241f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTDBR); 6242f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case SystemZ::LTXBRCompare_VecPseudo: 6243f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTXBR); 6244f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 62451d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand default: 62461d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand llvm_unreachable("Unexpected instr type to insert"); 62471d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand } 62481d09d56fe1e3f3faadd4bf4ccf3e585ddb3c3b07Ulrich Weigand} 6249