1f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 3f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// The LLVM Compiler Infrastructure 4f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 5f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// This file is distributed under the University of Illinois Open Source 6f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// License. See LICENSE.TXT for details. 7f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 8f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 9f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 10f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \file 11f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief This is the parent TargetLowering class for hardware code gen 12f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// targets. 13f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 14f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 15f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 16f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "AMDGPUISelLowering.h" 17e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "AMDGPU.h" 18a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard#include "AMDGPUFrameLowering.h" 19cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines#include "AMDGPUIntrinsicInfo.h" 2090c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPURegisterInfo.h" 2190c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUSubtarget.h" 22f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard#include "R600MachineFunctionInfo.h" 23e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "SIMachineFunctionInfo.h" 2436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Analysis/ValueTracking.h" 2590c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "llvm/CodeGen/CallingConvLower.h" 26f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineFunction.h" 27f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 28f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 29f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 30e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard#include "llvm/IR/DataLayout.h" 31dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/DiagnosticInfo.h" 32dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/DiagnosticPrinter.h" 33f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 34f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardusing namespace llvm; 35dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 36dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesnamespace { 37dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 38dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Diagnostic information for unimplemented or unsupported feature reporting. 39dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesclass DiagnosticInfoUnsupported : public DiagnosticInfo { 40dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesprivate: 41dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Twine &Description; 42dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Function &Fn; 43dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 44dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static int KindID; 45dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 46dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static int getKindID() { 47dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (KindID == 0) 48dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KindID = llvm::getNextAvailablePluginDiagnosticKind(); 49dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return KindID; 50dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 51dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 52dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinespublic: 53dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DiagnosticInfoUnsupported(const Function &Fn, const Twine &Desc, 54dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DiagnosticSeverity Severity = DS_Error) 55dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : DiagnosticInfo(getKindID(), Severity), 56dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Description(Desc), 57dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Fn(Fn) { } 58dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 59dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Function &getFunction() const { return Fn; } 60dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Twine &getDescription() const { return Description; } 61dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 62dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void print(DiagnosticPrinter &DP) const override { 63dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DP << "unsupported " << getDescription() << " in " << Fn.getName(); 64dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 65dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 66dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static bool classof(const DiagnosticInfo *DI) { 67dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DI->getKind() == getKindID(); 68dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 69dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 70dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 71dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesint DiagnosticInfoUnsupported::KindID = 0; 72dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 73dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 74dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 75f95b1621887e3409ceec2db47e1b44271d934735Tom Stellardstatic bool allocateStack(unsigned ValNo, MVT ValVT, MVT LocVT, 76f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard CCValAssign::LocInfo LocInfo, 77f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard ISD::ArgFlagsTy ArgFlags, CCState &State) { 7836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Offset = State.AllocateStack(ValVT.getStoreSize(), 7936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ArgFlags.getOrigAlign()); 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 81f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 82f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard return true; 83f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard} 84f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 8590c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUGenCallingConv.inc" 8690c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 87cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// Find a larger type to do a load / store of a vector with. 88cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesEVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { 89cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines unsigned StoreSize = VT.getStoreSizeInBits(); 90cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (StoreSize <= 32) 91cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return EVT::getIntegerVT(Ctx, StoreSize); 92cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 93cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines assert(StoreSize % 32 == 0 && "Store size not a multiple of 32"); 94cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); 95cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 96cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 97cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// Type for a vector that will be loaded to. 98cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesEVT AMDGPUTargetLowering::getEquivalentLoadRegType(LLVMContext &Ctx, EVT VT) { 99cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines unsigned StoreSize = VT.getStoreSizeInBits(); 100cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (StoreSize <= 32) 101cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return EVT::getIntegerVT(Ctx, 32); 102cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 103cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); 104cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 105cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 106f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardAMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) : 107f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TargetLowering(TM, new TargetLoweringObjectFileELF()) { 108f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 10936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Subtarget = &TM.getSubtarget<AMDGPUSubtarget>(); 11036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 111cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::Constant, MVT::i32, Legal); 112cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::Constant, MVT::i64, Legal); 113cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ConstantFP, MVT::f32, Legal); 114cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ConstantFP, MVT::f64, Legal); 115cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 116cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::BR_JT, MVT::Other, Expand); 117cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::BRIND, MVT::Other, Expand); 118f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 119f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // We need to custom lower some of the intrinsics 120f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 121f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 122f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Library functions. These default to Expand, but we have instructions 123f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // for them. 124f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FCEIL, MVT::f32, Legal); 125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FEXP2, MVT::f32, Legal); 126f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FPOW, MVT::f32, Legal); 127f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FLOG2, MVT::f32, Legal); 128f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FABS, MVT::f32, Legal); 129f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 130f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FRINT, MVT::f32, Legal); 1311afaeb1c39125115260b7a06b1dfc8f651d3ac2fBill Wendling setOperationAction(ISD::FROUND, MVT::f32, Legal); 13236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 133f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 134f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Lower floating point store/load to integer store/load to reduce the number 135f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // of patterns in tablegen. 136f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::f32, Promote); 137f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); 138f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 139fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard setOperationAction(ISD::STORE, MVT::v2f32, Promote); 140fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); 141fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard 142cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::STORE, MVT::i64, Promote); 143cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); 144cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 145f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::v4f32, Promote); 146f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); 147f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 148f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard setOperationAction(ISD::STORE, MVT::v8f32, Promote); 149f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); 150f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 151f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard setOperationAction(ISD::STORE, MVT::v16f32, Promote); 152f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); 153f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 15468e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::STORE, MVT::f64, Promote); 15568e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64); 15668e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::STORE, MVT::v2f64, Promote); 158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v2i64); 159dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1607a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // Custom lowering of vector stores is required for local address space 1617a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // stores. 1627a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard setOperationAction(ISD::STORE, MVT::v4i32, Custom); 1637a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // XXX: Native v2i32 local address space stores are possible, but not 1647a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // currently implemented. 1657a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard setOperationAction(ISD::STORE, MVT::v2i32, Custom); 1667a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 1674c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); 1684c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); 1694c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); 17036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1714c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard // XXX: This can be change to Custom, once ExpandVectorStores can 1724c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard // handle 64-bit stores. 1734c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); 1744c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::i64, MVT::i16, Expand); 176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::i64, MVT::i8, Expand); 17736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setTruncStoreAction(MVT::i64, MVT::i1, Expand); 17836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); 17936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setTruncStoreAction(MVT::v4i64, MVT::v4i1, Expand); 18036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 18136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 182f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::f32, Promote); 183f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); 184f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 185ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 186ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); 187ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard 188cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::LOAD, MVT::i64, Promote); 189cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); 190cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 191f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::v4f32, Promote); 192f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); 193f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 194f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard setOperationAction(ISD::LOAD, MVT::v8f32, Promote); 195f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); 196f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 197f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard setOperationAction(ISD::LOAD, MVT::v16f32, Promote); 198f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); 199f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 20068e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::LOAD, MVT::f64, Promote); 20168e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64); 20268e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::LOAD, MVT::v2f64, Promote); 204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v2i64); 205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 206a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 207a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); 20836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom); 20936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom); 210a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); 21136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); 21236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom); 21336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom); 21436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom); 21536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom); 216692ee102ebef535d311c35d53457028083e5c5beTom Stellard 21730d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v2i8, Expand); 21830d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v2i8, Expand); 21930d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i8, Expand); 22030d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v4i8, Expand); 22130d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v4i8, Expand); 22230d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i8, Expand); 22330d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, Expand); 22430d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, Expand); 22530d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, Expand); 22630d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, Expand); 22730d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, Expand); 22830d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, Expand); 22930d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard 23036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::BR_CC, MVT::i1, Expand); 23136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 232cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (Subtarget->getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) { 233cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FCEIL, MVT::f64, Custom); 234cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FTRUNC, MVT::f64, Custom); 235cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FRINT, MVT::f64, Custom); 236cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FFLOOR, MVT::f64, Custom); 237cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 239cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Subtarget->hasBFI()) { 240cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // fcopysign can be done in a single instruction with BFI. 241cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 242cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 243cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 244d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard 245cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 246cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (MVT VT : ScalarIntVTs) { 247cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SREM, VT, Expand); 248cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SDIV, VT, Expand); 249aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard 250cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // GPU does not have divrem function for signed or unsigned. 251cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SDIVREM, VT, Custom); 252cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UDIVREM, VT, Custom); 253cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 254cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // GPU does not have [S|U]MUL_LOHI functions as a single instruction. 255cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SMUL_LOHI, VT, Expand); 256cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UMUL_LOHI, VT, Expand); 25745b14e341a8a85e877d001bbd43f5e2b25b61cb8Christian Konig 258cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::BSWAP, VT, Expand); 259cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTTZ, VT, Expand); 260cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTLZ, VT, Expand); 261cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 262cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 263cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Subtarget->hasBCNT(32)) 264cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTPOP, MVT::i32, Expand); 265cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 266cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Subtarget->hasBCNT(64)) 267cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTPOP, MVT::i64, Expand); 268cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 269cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The hardware supports 32-bit ROTR, but not ROTL. 270cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ROTL, MVT::i32, Expand); 271cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ROTL, MVT::i64, Expand); 272cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ROTR, MVT::i64, Expand); 273cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 274cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FP_TO_SINT, MVT::i64, Expand); 275cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::MUL, MVT::i64, Expand); 276cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::MULHU, MVT::i64, Expand); 277cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::MULHS, MVT::i64, Expand); 278f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UDIV, MVT::i32, Expand); 279f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UREM, MVT::i32, Expand); 280cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 281cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); 282f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 283cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines static const MVT::SimpleValueType VectorIntTypes[] = { 2845464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::v2i32, MVT::v4i32 285f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry }; 286f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 287cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (MVT VT : VectorIntTypes) { 288cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Expand the following operations for the current type by default. 289f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::ADD, VT, Expand); 290f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::AND, VT, Expand); 291e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_SINT, VT, Expand); 292e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_UINT, VT, Expand); 293f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::MUL, VT, Expand); 294f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::OR, VT, Expand); 295f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SHL, VT, Expand); 296f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SRA, VT, Expand); 297cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SRL, VT, Expand); 298cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ROTL, VT, Expand); 299cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ROTR, VT, Expand); 300f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SUB, VT, Expand); 301cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SINT_TO_FP, VT, Expand); 302e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::UINT_TO_FP, VT, Expand); 303cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // TODO: Implement custom UREM / SREM routines. 304cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SDIV, VT, Expand); 305cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UDIV, VT, Expand); 306cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SREM, VT, Expand); 307f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::UREM, VT, Expand); 308cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SMUL_LOHI, VT, Expand); 309cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UMUL_LOHI, VT, Expand); 310cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SDIVREM, VT, Custom); 311cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::UDIVREM, VT, Custom); 312cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ADDC, VT, Expand); 313cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SUBC, VT, Expand); 314cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ADDE, VT, Expand); 315cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SUBE, VT, Expand); 31636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::SELECT, VT, Expand); 317f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, VT, Expand); 318cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SELECT_CC, VT, Expand); 319f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::XOR, VT, Expand); 320cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::BSWAP, VT, Expand); 321cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTPOP, VT, Expand); 322cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTTZ, VT, Expand); 323cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 324cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTLZ, VT, Expand); 325cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 326cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); 327f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry } 3280991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard 329cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines static const MVT::SimpleValueType FloatVectorTypes[] = { 3305464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::v2f32, MVT::v4f32 3310991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard }; 3320991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard 333cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (MVT VT : FloatVectorTypes) { 334ef39d3e9d0f82338406eb391ce67076eb2611565Bill Wendling setOperationAction(ISD::FABS, VT, Expand); 3350991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FADD, VT, Expand); 336cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FCEIL, VT, Expand); 337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOS, VT, Expand); 3380991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FDIV, VT, Expand); 339cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FEXP2, VT, Expand); 340cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FLOG2, VT, Expand); 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::FPOW, VT, Expand); 34284c0bd9803547b1cb09a20032bbc151a718b9457Tom Stellard setOperationAction(ISD::FFLOOR, VT, Expand); 34336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::FTRUNC, VT, Expand); 3440991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FMUL, VT, Expand); 345cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FMA, VT, Expand); 3463cae823f69f083c5eef19f0ea3128c48d7807c9bTom Stellard setOperationAction(ISD::FRINT, VT, Expand); 347cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FNEARBYINT, VT, Expand); 348f54a8409f9fcf025999d8a53fac989c6d0c82e72Tom Stellard setOperationAction(ISD::FSQRT, VT, Expand); 349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSIN, VT, Expand); 3500991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FSUB, VT, Expand); 351cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FNEG, VT, Expand); 35236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines setOperationAction(ISD::SELECT, VT, Expand); 353cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::VSELECT, VT, Expand); 354cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SELECT_CC, VT, Expand); 355cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FCOPYSIGN, VT, Expand); 356cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); 3570991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard } 35836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 359cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom); 360cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom); 361cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::MUL); 363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::SELECT_CC); 364cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 365cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setSchedulingPreference(Sched::RegPressure); 366cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setJumpIsExpensive(true); 367cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 368cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setSelectIsExpensive(false); 369cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines PredictableSelectIsExpensive = false; 370cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 371cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // There are no integer divide instructions, and these expand to a pretty 372cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // large sequence of instructions. 373cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setIntDivIsCheap(false); 374cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setPow2DivIsCheap(false); 375cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 376cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // TODO: Investigate this when 64-bit divides are implemented. 377cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines addBypassSlowDiv(64, 32); 378cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 379cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // FIXME: Need to really handle these. 380cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MaxStoresPerMemcpy = 4096; 381cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MaxStoresPerMemmove = 4096; 382cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MaxStoresPerMemset = 4096; 383f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 384f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 3852b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 3862b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard// Target Information 3872b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 3882b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 3892b272a1c8cb6d9f02223a598495d84cd9d75b13dTom StellardMVT AMDGPUTargetLowering::getVectorIdxTy() const { 3902b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard return MVT::i32; 3912b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard} 3922b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 393cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hinesbool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const { 394cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return true; 395cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 396cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 397cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// The backend supports 32 and 64 bit floating point immediates. 398cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// FIXME: Why are we reporting vectors of FP immediates as legal? 399cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hinesbool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 400cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT ScalarVT = VT.getScalarType(); 401cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64); 402cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 403cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 404cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// We don't want to shrink f64 / f32 constants. 405cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hinesbool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const { 406cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT ScalarVT = VT.getScalarType(); 407cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); 408cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 409cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 410509a492442b7e889d615d3b451629c81a810aef1Matt Arsenaultbool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, 411509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault EVT CastTy) const { 412509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault if (LoadTy.getSizeInBits() != CastTy.getSizeInBits()) 413509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault return true; 414509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault 415509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault unsigned LScalarSize = LoadTy.getScalarType().getSizeInBits(); 416509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault unsigned CastScalarSize = CastTy.getScalarType().getSizeInBits(); 417509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault 418509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault return ((LScalarSize <= CastScalarSize) || 419509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault (CastScalarSize >= 32) || 420509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault (LScalarSize < 32)); 421509a492442b7e889d615d3b451629c81a810aef1Matt Arsenault} 4222b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 423f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 4241f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard// Target Properties 4251f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 4261f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 4271f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { 4281f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 4291f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 4301f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 4311f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 4321f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFNegFree(EVT VT) const { 4331f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 4341f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 4351f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 4361f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const { 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Truncate is just accessing a subregister. 43936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0); 44036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 44136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const { 44336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Truncate is just accessing a subregister. 44436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Dest->getPrimitiveSizeInBits() < Source->getPrimitiveSizeInBits() && 44536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (Dest->getPrimitiveSizeInBits() % 32 == 0); 44636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const { 44936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DataLayout *DL = getDataLayout(); 45036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned SrcSize = DL->getTypeSizeInBits(Src->getScalarType()); 45136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned DestSize = DL->getTypeSizeInBits(Dest->getScalarType()); 45236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SrcSize == 32 && DestSize == 64; 45436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 45536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const { 45736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Any register load of a 64-bit value really requires 2 32-bit moves. For all 45836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // practical purposes, the extra mov 0 to load a 64-bit is free. As used, 45936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this will enable reducing 64-bit operations the 32-bit, which is always 46036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // good. 46136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Src == MVT::i32 && Dest == MVT::i64; 46236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 46336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 464cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hinesbool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 465cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return isZExtFree(Val.getValueType(), VT2); 466cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 467cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 46836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const { 46936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // There aren't really 64-bit registers, but pairs of 32-bit ones and only a 47036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // limited number of native 64-bit operations. Shrinking an operation to fit 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // in a single 32-bit register should always be helpful. As currently used, 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this is much less general than the name suggests, and is only used in 47336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // not profitable, and may actually be harmful. 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32; 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4781f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 479f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// TargetLowering Callbacks 480f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 481f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 48290c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konigvoid AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, 48390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig const SmallVectorImpl<ISD::InputArg> &Ins) const { 48490c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 48590c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig State.AnalyzeFormalArguments(Ins, CC_AMDGPU); 486f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 487f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 488f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerReturn( 489f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain, 490f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CallingConv::ID CallConv, 491f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard bool isVarArg, 492f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<ISD::OutputArg> &Outs, 493f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<SDValue> &OutVals, 494ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, SelectionDAG &DAG) const { 495f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); 496f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 497f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 498f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 499f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Target specific lowering 500f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 501f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 502dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI, 503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVectorImpl<SDValue> &InVals) const { 504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Callee = CLI.Callee; 505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = CLI.DAG; 506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Function &Fn = *DAG.getMachineFunction().getFunction(); 508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines StringRef FuncName("<unknown>"); 510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee)) 512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncName = G->getSymbol(); 513dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 514dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncName = G->getGlobal()->getName(); 515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DiagnosticInfoUnsupported NoCalls(Fn, "call to function " + FuncName); 517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getContext()->diagnose(NoCalls); 518dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 519dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 521cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, 522cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SelectionDAG &DAG) const { 523f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Op.getOpcode()) { 524f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: 525f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getNode()->dump(); 52636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Custom lowering code for this" 52736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "instruction is not implemented yet!"); 528f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 529f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 530a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 531a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 532a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard case ISD::FrameIndex: return LowerFrameIndex(Op, DAG); 533f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 534cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SDIV: return LowerSDIV(Op, DAG); 535cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SREM: return LowerSREM(Op, DAG); 536f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 537cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); 538cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FCEIL: return LowerFCEIL(Op, DAG); 539cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); 540cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FRINT: return LowerFRINT(Op, DAG); 541cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); 542cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); 543aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 544f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 545f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Op; 546f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 547f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 54836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N, 54936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<SDValue> &Results, 55036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 55136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (N->getOpcode()) { 55236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::SIGN_EXTEND_INREG: 55336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Different parts of legalization seem to interpret which type of 55436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // sign_extend_inreg is the one to check for custom lowering. The extended 55536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // from type is what really matters, but some places check for custom 55636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // lowering of the result type. This results in trying to use 55736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do 55836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // nothing here and let the illegal result integer be handled normally. 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 560cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::LOAD: { 561cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDNode *Node = LowerLOAD(SDValue(N, 0), DAG).getNode(); 562cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Node) 563cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return; 564cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 565cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(SDValue(Node, 0)); 566cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(SDValue(Node, 1)); 567cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // XXX: LLVM seems not to replace Chain Value inside CustomWidenLowerNode 568cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // function 569cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.ReplaceAllUsesOfValueWith(SDValue(N,1), SDValue(Node, 1)); 570cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return; 571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 572cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::STORE: { 573cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Lowered = LowerSTORE(SDValue(N, 0), DAG); 574cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (Lowered.getNode()) 575cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(Lowered); 576cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return; 577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 57836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 57936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 58036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 58136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 58236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// FIXME: This implements accesses to initialized globals in the constant 584dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// address space by copying them to private and accessing that. It does not 585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// properly handle illegal types or vectors. The private vector loads are not 586dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// scalarized, and the illegal scalars hit an assertion. This technique will not 587dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// work well with large initializers, and this should eventually be 588dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// removed. Initialized globals should be placed into a data section that the 589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// runtime will load into a buffer before the kernel is executed. Uses of the 590dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// global need to be replaced with a pointer loaded from an implicit kernel 591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// argument into this buffer holding the copy of the data, which will remove the 592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// need for any of this. 59336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init, 59436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const GlobalValue *GV, 59536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SDValue &InitPtr, 59636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Chain, 59736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 59836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DataLayout *TD = getTargetMachine().getDataLayout(); 59936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(InitPtr); 600cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Type *InitTy = Init->getType(); 601cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 60236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const ConstantInt *CI = dyn_cast<ConstantInt>(Init)) { 603cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = EVT::getEVT(InitTy); 604cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); 605cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getStore(Chain, DL, DAG.getConstant(*CI, VT), InitPtr, 606cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 607cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines TD->getPrefTypeAlignment(InitTy)); 608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Init)) { 61136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = EVT::getEVT(CFP->getType()); 61236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PointerType *PtrTy = PointerType::get(CFP->getType(), 0); 61336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, VT), InitPtr, 61436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 61536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TD->getPrefTypeAlignment(CFP->getType())); 616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (StructType *ST = dyn_cast<StructType>(InitTy)) { 619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const StructLayout *SL = TD->getStructLayout(ST); 620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 62136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT PtrVT = InitPtr.getValueType(); 622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> Chains; 623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned I = 0, N = ST->getNumElements(); I != N; ++I) { 625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Offset = DAG.getConstant(SL->getElementOffset(I), PtrVT); 626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); 627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Constant *Elt = Init->getAggregateElement(I); 629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); 630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SequentialType *SeqTy = dyn_cast<SequentialType>(InitTy)) { 636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT PtrVT = InitPtr.getValueType(); 637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElements; 639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ArrayType *AT = dyn_cast<ArrayType>(SeqTy)) 640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumElements = AT->getNumElements(); 641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (VectorType *VT = dyn_cast<VectorType>(SeqTy)) 642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumElements = VT->getNumElements(); 643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unexpected type"); 645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned EltSize = TD->getTypeAllocSize(SeqTy->getElementType()); 64736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<SDValue, 8> Chains; 64836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0; i < NumElements; ++i) { 649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Offset = DAG.getConstant(i * EltSize, PtrVT); 65036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); 651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Constant *Elt = Init->getAggregateElement(i); 653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); 65436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 65736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 659cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (isa<UndefValue>(Init)) { 660cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = EVT::getEVT(InitTy); 661cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); 662cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr, 663cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 664cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines TD->getPrefTypeAlignment(InitTy)); 665cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 666cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Init->dump(); 668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unhandled constant initializer"); 66936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 67036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 671e3d4cbc7d25061441adafa47450a31571c87bf85Tom StellardSDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, 672e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SDValue Op, 673e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SelectionDAG &DAG) const { 674e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 675e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard const DataLayout *TD = getTargetMachine().getDataLayout(); 676e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op); 67736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const GlobalValue *GV = G->getGlobal(); 678da25cd3e6de8f21005590c2de49868f883cf2410Tom Stellard 67936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (G->getAddressSpace()) { 68036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: llvm_unreachable("Global Address lowering not implemented for this " 68136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "address space"); 68236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUAS::LOCAL_ADDRESS: { 68336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // XXX: What does the value of G->getOffset() mean? 68436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(G->getOffset() == 0 && 685e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard "Do not know what to do with an non-zero offset"); 686e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 68736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Offset; 68836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MFI->LocalMemoryObjects.count(GV) == 0) { 68936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType()); 69036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Offset = MFI->LDSSize; 69136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MFI->LocalMemoryObjects[GV] = Offset; 69236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // XXX: Account for alignment? 69336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MFI->LDSSize += Size; 69436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 69536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Offset = MFI->LocalMemoryObjects[GV]; 69636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 697e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 69836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getConstant(Offset, getPointerTy(G->getAddressSpace())); 69936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 70036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUAS::CONSTANT_ADDRESS: { 70136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 70236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *EltType = GV->getType()->getElementType(); 70336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Size = TD->getTypeAllocSize(EltType); 70436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Alignment = TD->getPrefTypeAlignment(EltType); 70536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 706cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MVT PrivPtrVT = getPointerTy(AMDGPUAS::PRIVATE_ADDRESS); 707cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MVT ConstPtrVT = getPointerTy(AMDGPUAS::CONSTANT_ADDRESS); 708cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 709cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int FI = FrameInfo->CreateStackObject(Size, Alignment, false); 710cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT); 711cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalVariable *Var = cast<GlobalVariable>(GV); 713cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Var->hasInitializer()) { 714cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // This has no use, but bugpoint will hit it. 715cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT); 716cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 717cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 71836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Constant *Init = Var->getInitializer(); 71936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<SDNode*, 8> WorkList; 72036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 72136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(), 72236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = DAG.getEntryNode()->use_end(); I != E; ++I) { 72336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I->getOpcode() != AMDGPUISD::REGISTER_LOAD && I->getOpcode() != ISD::LOAD) 72436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 72536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.push_back(*I); 72636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 72736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG); 72836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVector<SDNode*, 8>::iterator I = WorkList.begin(), 72936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = WorkList.end(); I != E; ++I) { 73036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<SDValue, 8> Ops; 73136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ops.push_back(Chain); 73236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) { 73336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ops.push_back((*I)->getOperand(i)); 73436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.UpdateNodeOperands(*I, Ops); 73636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 737cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT); 73836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 739470c451574609adcaab9b279cc74c6ff0f91b00fTom Stellard } 740e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard} 741e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 742a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op, 743a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 744a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 745a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue A = Op.getOperand(0); 746a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue B = Op.getOperand(1); 747a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.ExtractVectorElements(A, Args); 749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.ExtractVectorElements(B, Args); 750a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args); 752a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 753a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 754a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 755a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 756a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 757a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 758a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.ExtractVectorElements(Op.getOperand(0), Args, Start, 761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT.getVectorNumElements()); 762a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args); 764a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 765a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 766a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom StellardSDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op, 767a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard SelectionDAG &DAG) const { 768a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard 769a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 770a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard const AMDGPUFrameLowering *TFL = 771a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard static_cast<const AMDGPUFrameLowering*>(getTargetMachine().getFrameLowering()); 772a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard 773cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op); 774a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard 775a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard unsigned FrameIndex = FIN->getIndex(); 776a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard unsigned Offset = TFL->getFrameIndexOffset(MF, FrameIndex); 777a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), 778a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard Op.getValueType()); 779a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard} 780a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 781f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 782f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 783f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 784ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 785f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 786f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 787f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (IntrinsicID) { 788f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return Op; 789cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_abs: 790cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDIL_abs: // Legacy name. 791f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicIABS(Op, DAG); 792f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_lrp: 793f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicLRP(Op, DAG); 794cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_fract: 795cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDIL_fraction: // Legacy name. 796f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 797cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 798cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_clamp: 799cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDIL_clamp: // Legacy name. 800cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::CLAMP, DL, VT, 801cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 802cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 803cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_div_scale: { 804cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // 3rd parameter required to be a constant. 805cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const ConstantSDNode *Param = dyn_cast<ConstantSDNode>(Op.getOperand(3)); 806cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!Param) 807cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getUNDEF(VT); 808cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 809cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Translate to the operands expected by the machine instruction. The 810cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // first parameter must be the same as the first instruction. 811cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Numerator = Op.getOperand(1); 812cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Denominator = Op.getOperand(2); 813cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator; 814cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 815cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, VT, 816cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Src0, Denominator, Numerator); 817cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 818cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 819cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_div_fmas: 820cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, 821cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 822cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 823cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_div_fixup: 824cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT, 825cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 826cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 827cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_trig_preop: 828cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::TRIG_PREOP, DL, VT, 829cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Op.getOperand(1), Op.getOperand(2)); 830cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 831cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_rcp: 832cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1)); 833cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 834cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_rsq: 835cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 836cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 837cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_legacy_rsq: 838cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 839cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 840cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_rsq_clamped: 841cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::RSQ_CLAMPED, DL, VT, Op.getOperand(1)); 842cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 843f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imax: 844f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1), 845f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 846f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umax: 847f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1), 848f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 849f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imin: 850f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1), 851f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 852f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umin: 853f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1), 854f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 85536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUIntrinsic::AMDGPU_umul24: 857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT, 858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1), Op.getOperand(2)); 859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUIntrinsic::AMDGPU_imul24: 861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT, 862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1), Op.getOperand(2)); 863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUIntrinsic::AMDGPU_umad24: 865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::MAD_U24, DL, VT, 866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUIntrinsic::AMDGPU_imad24: 869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::MAD_I24, DL, VT, 870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 872cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte0: 873cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Op.getOperand(1)); 874cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 875cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte1: 876cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE1, DL, VT, Op.getOperand(1)); 877cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 878cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte2: 879cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE2, DL, VT, Op.getOperand(1)); 880cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 881cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte3: 882cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE3, DL, VT, Op.getOperand(1)); 883cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 88436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_bfe_i32: 88536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT, 88636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(1), 88736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(2), 88836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(3)); 88936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 89036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_bfe_u32: 89136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT, 89236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(1), 89336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(2), 89436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(3)); 89536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 89636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_bfi: 89736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(AMDGPUISD::BFI, DL, VT, 89836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(1), 89936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(2), 90036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(3)); 90136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 90236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_bfm: 90336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(AMDGPUISD::BFM, DL, VT, 90436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(1), 90536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(2)); 90636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 907cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_brev: 908cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::BREV, DL, VT, Op.getOperand(1)); 909cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 910cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDIL_exp: // Legacy name. 911cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1)); 912cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 913cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDIL_round_nearest: // Legacy name. 914f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1)); 915cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case AMDGPUIntrinsic::AMDGPU_trunc: // Legacy name. 916cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::FTRUNC, DL, VT, Op.getOperand(1)); 917f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 918f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 919f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 920f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard///IABS(a) = SMAX(sub(0, a), a) 921f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op, 922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 923ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 924f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 925f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 926f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 927f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 928f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1)); 929f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 930f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 931f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// Linear Interpolation 932f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// LRP(a, b, c) = muladd(a, b, (1 - a) * c) 933f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op, 934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 935ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 936f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 937f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT, 938f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstantFP(1.0f, MVT::f32), 939f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 940f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA, 941f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(3)); 942e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune return DAG.getNode(ISD::FADD, DL, VT, 943e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)), 944e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune OneSubAC); 945f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 946f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 947f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief Generate Min/Max node 948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AMDGPUTargetLowering::CombineMinMax(SDNode *N, 949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 952f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = N->getOperand(0); 954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = N->getOperand(1); 955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue True = N->getOperand(2); 956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue False = N->getOperand(3); 957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CC = N->getOperand(4); 958f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 959f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (VT != MVT::f32 || 960f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard !((LHS == True && RHS == False) || (LHS == False && RHS == True))) { 961f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return SDValue(); 962f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 963f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 964f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 965f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (CCOpcode) { 966f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOEQ: 967f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETONE: 968f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUNE: 969f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETNE: 970f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUEQ: 971f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETEQ: 972f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE: 973f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE2: 974f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE: 975f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE2: 976f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUO: 977f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETO: 97836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Operation should already be optimised!"); 979f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULE: 980f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULT: 981f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLE: 982f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLT: 983f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLE: 984f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLT: { 985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (LHS == True) ? AMDGPUISD::FMIN : AMDGPUISD::FMAX; 986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, DL, VT, LHS, RHS); 987f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 988f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGT: 989f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGE: 990f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGE: 991f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGE: 992f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGT: 993f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGT: { 994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (LHS == True) ? AMDGPUISD::FMAX : AMDGPUISD::FMIN; 995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, DL, VT, LHS, RHS); 996f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 997f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETCC_INVALID: 99836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Invalid setcc condcode!"); 999f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 1001f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1002f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1003d08a9303614355cfdcac5f2c27c09ce809565423Tom StellardSDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue &Op, 1004d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SelectionDAG &DAG) const { 1005d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard LoadSDNode *Load = dyn_cast<LoadSDNode>(Op); 1006d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT MemEltVT = Load->getMemoryVT().getVectorElementType(); 1007cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT LoadVT = Op.getValueType(); 1008d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT EltVT = Op.getValueType().getVectorElementType(); 1009d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT PtrVT = Load->getBasePtr().getValueType(); 1010cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1011d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard unsigned NumElts = Load->getMemoryVT().getVectorNumElements(); 1012d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SmallVector<SDValue, 8> Loads; 1013cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SmallVector<SDValue, 8> Chains; 1014cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1015d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDLoc SL(Op); 1016d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 1017d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard for (unsigned i = 0, e = NumElts; i != e; ++i) { 1018d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(), 1019d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), PtrVT)); 1020cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1021cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue NewLoad 1022cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines = DAG.getExtLoad(Load->getExtensionType(), SL, EltVT, 1023cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Load->getChain(), Ptr, 1024cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MachinePointerInfo(Load->getMemOperand()->getValue()), 1025cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MemEltVT, Load->isVolatile(), Load->isNonTemporal(), 1026cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Load->getAlignment()); 1027cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Loads.push_back(NewLoad.getValue(0)); 1028cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Chains.push_back(NewLoad.getValue(1)); 1029d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard } 1030cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1031cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Ops[] = { 1032cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::BUILD_VECTOR, SL, LoadVT, Loads), 1033cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains) 1034cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1035cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1036cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Ops, SL); 1037d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard} 1038d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 10397a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op, 10407a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SelectionDAG &DAG) const { 1041cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines StoreSDNode *Store = cast<StoreSDNode>(Op); 10427a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemVT = Store->getMemoryVT(); 10437a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemBits = MemVT.getSizeInBits(); 10447a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 104536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Byte stores are really expensive, so if possible, try to pack 32-bit vector 104636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // truncating store into an i32 store. 104736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // XXX: We could also handle optimize other vector bitwidths. 10487a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (!MemVT.isVector() || MemBits > 32) { 10497a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SDValue(); 10507a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 10517a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 10527a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDLoc DL(Op); 1053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Value = Store->getValue(); 10547a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT VT = Value.getValueType(); 1055dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ElemVT = VT.getVectorElementType(); 1056dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ptr = Store->getBasePtr(); 10577a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemEltVT = MemVT.getVectorElementType(); 10587a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemEltBits = MemEltVT.getSizeInBits(); 10597a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemNumElements = MemVT.getVectorNumElements(); 1060dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PackedSize = MemVT.getStoreSizeInBits(); 1061dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, MVT::i32); 1062dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Value.getValueType().getScalarSizeInBits() >= 32); 106436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10657a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue PackedValue; 10667a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard for (unsigned i = 0; i < MemNumElements; ++i) { 10677a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value, 10687a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard DAG.getConstant(i, MVT::i32)); 1069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32); 1070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg 1071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Shift = DAG.getConstant(MemEltBits * i, MVT::i32); 1073dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift); 1074dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10757a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (i == 0) { 10767a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard PackedValue = Elt; 10777a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } else { 1078dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt); 10797a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 10807a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 1081dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (PackedSize < 32) { 1083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize); 1084dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr, 1085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Store->getMemOperand()->getPointerInfo(), 1086dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PackedVT, 1087dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Store->isNonTemporal(), Store->isVolatile(), 1088dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Store->getAlignment()); 1089dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1090dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10917a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr, 1092dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Store->getMemOperand()->getPointerInfo(), 10937a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->isVolatile(), Store->isNonTemporal(), 10947a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getAlignment()); 10957a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 10967a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 10977a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op, 10987a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SelectionDAG &DAG) const { 10997a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard StoreSDNode *Store = cast<StoreSDNode>(Op); 11007a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemEltVT = Store->getMemoryVT().getVectorElementType(); 11017a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT EltVT = Store->getValue().getValueType().getVectorElementType(); 11027a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT PtrVT = Store->getBasePtr().getValueType(); 11037a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned NumElts = Store->getMemoryVT().getVectorNumElements(); 11047a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDLoc SL(Op); 11057a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 11067a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SmallVector<SDValue, 8> Chains; 11077a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 11087a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard for (unsigned i = 0, e = NumElts; i != e; ++i) { 11097a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 11107a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getValue(), DAG.getConstant(i, MVT::i32)); 11117a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, 11127a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getBasePtr(), 11137a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), 11147a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard PtrVT)); 11158e78012457682d335ee97cf2859dfe03b7e2ae93Tom Stellard Chains.push_back(DAG.getTruncStore(Store->getChain(), SL, Val, Ptr, 11167a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard MachinePointerInfo(Store->getMemOperand()->getValue()), 11178e78012457682d335ee97cf2859dfe03b7e2ae93Tom Stellard MemEltVT, Store->isVolatile(), Store->isNonTemporal(), 11187a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getAlignment())); 11197a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 1120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains); 11217a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 11227a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 112336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 112436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 112536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LoadSDNode *Load = cast<LoadSDNode>(Op); 112636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ISD::LoadExtType ExtType = Load->getExtensionType(); 112736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = Op.getValueType(); 112836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT MemVT = Load->getMemoryVT(); 112936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 113036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ExtType != ISD::NON_EXTLOAD && !VT.isVector() && VT.getSizeInBits() > 32) { 113136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We can do the extload to 32-bits, and then need to separately extend to 113236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 64-bits. 113336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 113436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ExtLoad32 = DAG.getExtLoad(ExtType, DL, MVT::i32, 113536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getChain(), 113636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getBasePtr(), 113736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemVT, 113836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getMemOperand()); 1139cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1140cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Ops[] = { 1141cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::getExtForLoadExtType(ExtType), DL, VT, ExtLoad32), 1142cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ExtLoad32.getValue(1) 1143cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1144cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1145cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Ops, DL); 114636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 114736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1148dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ExtType == ISD::NON_EXTLOAD && VT.getSizeInBits() < 32) { 1149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VT == MVT::i1 && "Only i1 non-extloads expected"); 1150dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: Copied from PPC 1151dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // First, load into 32 bits, then truncate to 1 bit. 1152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = Load->getChain(); 1154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BasePtr = Load->getBasePtr(); 1155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineMemOperand *MMO = Load->getMemOperand(); 1156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, 1158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasePtr, MVT::i8, MMO); 1159cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1160cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Ops[] = { 1161cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD), 1162cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NewLD.getValue(1) 1163cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1164cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1165cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Ops, DL); 1166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 116836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower loads constant address space global variable loads 116936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS && 1170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<GlobalVariable>( 1171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GetUnderlyingObject(Load->getMemOperand()->getValue()))) { 117236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1173cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 117436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL, 117536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines getPointerTy(AMDGPUAS::PRIVATE_ADDRESS)); 117636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 117736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(2, MVT::i32)); 1178cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op->getVTList(), 117936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getChain(), Ptr, 118036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getTargetConstant(0, MVT::i32), Op.getOperand(2)); 118136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 118236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 118336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Load->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS || 118436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32)) 118536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SDValue(); 118636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 118736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 118836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(), 118936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(2, MVT::i32)); 119036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(), 119136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getChain(), Ptr, 119236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getTargetConstant(0, MVT::i32), 119336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op.getOperand(2)); 119436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, 119536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Load->getBasePtr(), 119636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(0x3, MVT::i32)); 119736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 119836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(3, MVT::i32)); 119936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 120036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt); 120136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 120236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT MemEltVT = MemVT.getScalarType(); 120336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ExtType == ISD::SEXTLOAD) { 120436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue MemEltVTNode = DAG.getValueType(MemEltVT); 1205cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1206cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Ops[] = { 1207cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode), 1208cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Load->getChain() 1209cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1210cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1211cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Ops, DL); 121236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 121336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1214cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Ops[] = { 1215cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getZeroExtendInReg(Ret, DL, MemEltVT), 1216cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Load->getChain() 1217cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1218cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1219cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Ops, DL); 122036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 122136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12227a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 122336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 12247a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG); 12257a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (Result.getNode()) { 12267a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return Result; 12277a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 1228f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 12297a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard StoreSDNode *Store = cast<StoreSDNode>(Op); 123036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Chain = Store->getChain(); 1231a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard if ((Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || 1232a2b4eb6d15a13de257319ac6231b5ab622cd02b1Tom Stellard Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) && 12337a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getValue().getValueType().isVector()) { 12347a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SplitVectorStore(Op, DAG); 12357a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 123636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 123736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT MemVT = Store->getMemoryVT(); 123836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS && 123936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemVT.bitsLT(MVT::i32)) { 124036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Mask = 0; 124136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Store->getMemoryVT() == MVT::i8) { 124236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Mask = 0xff; 124336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (Store->getMemoryVT() == MVT::i16) { 124436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Mask = 0xffff; 124536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 124636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue BasePtr = Store->getBasePtr(); 124736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr, 124836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(2, MVT::i32)); 124936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32, 125036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Chain, Ptr, DAG.getTargetConstant(0, MVT::i32)); 125136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 125236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr, 125336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(0x3, MVT::i32)); 125436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 125536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 125636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(3, MVT::i32)); 125736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 125836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, 125936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Store->getValue()); 126036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 126136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT); 126236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 126336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, 126436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MaskedValue, ShiftAmt); 126536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 126636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, DAG.getConstant(Mask, MVT::i32), 126736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ShiftAmt); 126836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask, 126936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DAG.getConstant(0xffffffff, MVT::i32)); 127036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); 127136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 127236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); 127336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 127436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Chain, Value, Ptr, DAG.getTargetConstant(0, MVT::i32)); 127536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 12767a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SDValue(); 12777a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 1278f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1279cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSDIV24(SDValue Op, SelectionDAG &DAG) const { 1280cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1281cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT OVT = Op.getValueType(); 1282cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS = Op.getOperand(0); 1283cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS = Op.getOperand(1); 1284cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MVT INTTY; 1285cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines MVT FLTTY; 1286cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (!OVT.isVector()) { 1287cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines INTTY = MVT::i32; 1288cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines FLTTY = MVT::f32; 1289cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else if (OVT.getVectorNumElements() == 2) { 1290cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines INTTY = MVT::v2i32; 1291cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines FLTTY = MVT::v2f32; 1292cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else if (OVT.getVectorNumElements() == 4) { 1293cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines INTTY = MVT::v4i32; 1294cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines FLTTY = MVT::v4f32; 1295cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 1296cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines unsigned bitsize = OVT.getScalarType().getSizeInBits(); 1297cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // char|short jq = ia ^ ib; 1298cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue jq = DAG.getNode(ISD::XOR, DL, OVT, LHS, RHS); 1299cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1300cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // jq = jq >> (bitsize - 2) 1301cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines jq = DAG.getNode(ISD::SRA, DL, OVT, jq, DAG.getConstant(bitsize - 2, OVT)); 1302cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1303cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // jq = jq | 0x1 1304cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines jq = DAG.getNode(ISD::OR, DL, OVT, jq, DAG.getConstant(1, OVT)); 1305cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1306cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // jq = (int)jq 1307cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines jq = DAG.getSExtOrTrunc(jq, DL, INTTY); 1308cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1309cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // int ia = (int)LHS; 1310cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ia = DAG.getSExtOrTrunc(LHS, DL, INTTY); 1311cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1312cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // int ib, (int)RHS; 1313cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ib = DAG.getSExtOrTrunc(RHS, DL, INTTY); 1314cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1315cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // float fa = (float)ia; 1316cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue fa = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ia); 1317cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1318cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // float fb = (float)ib; 1319cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue fb = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ib); 1320cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1321cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // float fq = native_divide(fa, fb); 1322cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue fq = DAG.getNode(ISD::FMUL, DL, FLTTY, 1323cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines fa, DAG.getNode(AMDGPUISD::RCP, DL, FLTTY, fb)); 1324cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1325cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // fq = trunc(fq); 1326cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines fq = DAG.getNode(ISD::FTRUNC, DL, FLTTY, fq); 1327cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1328cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // float fqneg = -fq; 1329cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FLTTY, fq); 1330cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1331cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // float fr = mad(fqneg, fb, fa); 1332cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue fr = DAG.getNode(ISD::FADD, DL, FLTTY, 1333cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::MUL, DL, FLTTY, fqneg, fb), fa); 1334cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1335cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // int iq = (int)fq; 1336cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue iq = DAG.getNode(ISD::FP_TO_SINT, DL, INTTY, fq); 1337cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1338cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // fr = fabs(fr); 1339cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines fr = DAG.getNode(ISD::FABS, DL, FLTTY, fr); 1340cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1341cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // fb = fabs(fb); 1342cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines fb = DAG.getNode(ISD::FABS, DL, FLTTY, fb); 1343cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1344cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // int cv = fr >= fb; 1345cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue cv; 1346cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (INTTY == MVT::i32) { 1347cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE); 1348cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else { 1349cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE); 1350cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 1351cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // jq = (cv ? jq : 0); 1352cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines jq = DAG.getNode(ISD::SELECT, DL, OVT, cv, jq, 1353cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(0, OVT)); 1354cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // dst = iq + jq; 1355cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines iq = DAG.getSExtOrTrunc(iq, DL, OVT); 1356cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines iq = DAG.getNode(ISD::ADD, DL, OVT, iq, jq); 1357cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return iq; 1358cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1359cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1360cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSDIV32(SDValue Op, SelectionDAG &DAG) const { 1361cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1362cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT OVT = Op.getValueType(); 1363cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS = Op.getOperand(0); 1364cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS = Op.getOperand(1); 1365cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The LowerSDIV32 function generates equivalent to the following IL. 1366cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r0, LHS 1367cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r1, RHS 1368cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r10, r0, 0 1369cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r11, r1, 0 1370cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1371cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r1, r1, r11 1372cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r0, r0, r10 1373cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r1, r1, r11 1374cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // udiv r0, r0, r1 1375cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r10, r10, r11 1376cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1377cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor DST, r0, r10 1378cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1379cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r0, LHS 1380cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r0 = LHS; 1381cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1382cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r1, RHS 1383cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r1 = RHS; 1384cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1385cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r10, r0, 0 1386cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r10 = DAG.getSelectCC(DL, 1387cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0, DAG.getConstant(0, OVT), 1388cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(-1, OVT), 1389cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(0, OVT), 1390cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ISD::SETLT); 1391cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1392cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r11, r1, 0 1393cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r11 = DAG.getSelectCC(DL, 1394cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r1, DAG.getConstant(0, OVT), 1395cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(-1, OVT), 1396cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(0, OVT), 1397cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ISD::SETLT); 1398cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1399cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1400cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10); 1401cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1402cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r1, r1, r11 1403cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11); 1404cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1405cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r0, r0, r10 1406cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10); 1407cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1408cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r1, r1, r11 1409cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11); 1410cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1411cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // udiv r0, r0, r1 1412cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::UDIV, DL, OVT, r0, r1); 1413cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1414cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r10, r10, r11 1415cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r10 = DAG.getNode(ISD::XOR, DL, OVT, r10, r11); 1416cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1417cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1418cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10); 1419cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1420cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor DST, r0, r10 1421cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10); 1422cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DST; 1423cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1424cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1425cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSDIV64(SDValue Op, SelectionDAG &DAG) const { 1426cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(Op.getNode(), 0); 1427cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1428cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1429cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSDIV(SDValue Op, SelectionDAG &DAG) const { 1430cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT OVT = Op.getValueType().getScalarType(); 1431cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1432cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (OVT == MVT::i64) 1433cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return LowerSDIV64(Op, DAG); 1434cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1435cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (OVT.getScalarType() == MVT::i32) 1436cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return LowerSDIV32(Op, DAG); 1437cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1438cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (OVT == MVT::i16 || OVT == MVT::i8) { 1439cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // FIXME: We should be checking for the masked bits. This isn't reached 1440cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // because i8 and i16 are not legal types. 1441cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return LowerSDIV24(Op, DAG); 1442cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 1443cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1444cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(Op.getNode(), 0); 1445cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1446cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1447cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSREM32(SDValue Op, SelectionDAG &DAG) const { 1448cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1449cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT OVT = Op.getValueType(); 1450cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS = Op.getOperand(0); 1451cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS = Op.getOperand(1); 1452cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The LowerSREM32 function generates equivalent to the following IL. 1453cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r0, LHS 1454cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r1, RHS 1455cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r10, r0, 0 1456cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r11, r1, 0 1457cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1458cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r1, r1, r11 1459cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r0, r0, r10 1460cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r1, r1, r11 1461cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // udiv r20, r0, r1 1462cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // umul r20, r20, r1 1463cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // sub r0, r0, r20 1464cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1465cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor DST, r0, r10 1466cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1467cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r0, LHS 1468cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r0 = LHS; 1469cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1470cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // mov r1, RHS 1471cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r1 = RHS; 1472cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1473cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r10, r0, 0 1474cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r10 = DAG.getSetCC(DL, OVT, r0, DAG.getConstant(0, OVT), ISD::SETLT); 1475cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1476cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ilt r11, r1, 0 1477cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r11 = DAG.getSetCC(DL, OVT, r1, DAG.getConstant(0, OVT), ISD::SETLT); 1478cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1479cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1480cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10); 1481cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1482cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r1, r1, r11 1483cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11); 1484cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1485cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r0, r0, r10 1486cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10); 1487cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1488cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor r1, r1, r11 1489cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11); 1490cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1491cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // udiv r20, r0, r1 1492cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue r20 = DAG.getNode(ISD::UREM, DL, OVT, r0, r1); 1493cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1494cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // umul r20, r20, r1 1495cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r20 = DAG.getNode(AMDGPUISD::UMUL, DL, OVT, r20, r1); 1496cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1497cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // sub r0, r0, r20 1498cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::SUB, DL, OVT, r0, r20); 1499cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1500cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // iadd r0, r0, r10 1501cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10); 1502cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1503cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ixor DST, r0, r10 1504cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10); 1505cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DST; 1506cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1507cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1508cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSREM64(SDValue Op, SelectionDAG &DAG) const { 1509cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(Op.getNode(), 0); 1510cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1511cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1512cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSREM(SDValue Op, SelectionDAG &DAG) const { 1513cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT OVT = Op.getValueType(); 1514cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1515cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (OVT.getScalarType() == MVT::i64) 1516cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return LowerSREM64(Op, DAG); 1517cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1518cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (OVT.getScalarType() == MVT::i32) 1519cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return LowerSREM32(Op, DAG); 1520cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1521cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(Op.getNode(), 0); 1522cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1523cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1524f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 1525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 1526ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 1527f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 1528f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1529f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num = Op.getOperand(0); 1530f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Den = Op.getOperand(1); 1531f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1532f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP = URECIP(Den) = 2^32 / Den + e 1533f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // e is rounding error. 1534f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); 1535f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1536f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_LO = umulo(RCP, Den) */ 1537f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den); 1538f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1539f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_HI = mulhu (RCP, Den) */ 1540f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); 1541f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1542f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // NEG_RCP_LO = -RCP_LO 1543f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 1544f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_LO); 1545f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1546f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO) 1547f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 1548f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NEG_RCP_LO, RCP_LO, 1549f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 1550f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate the rounding error from the URECIP instruction 1551f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // E = mulhu(ABS_RCP_LO, RCP) 1552f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); 1553f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1554f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_A_E = RCP + E 1555f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); 1556f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1557f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_S_E = RCP - E 1558f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); 1559f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1560f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E) 1561f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 1562f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_A_E, RCP_S_E, 1563f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 1564f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient = mulhu(Tmp0, Num) 1565f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); 1566f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1567f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Num_S_Remainder = Quotient * Den 1568f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den); 1569f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1570f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder = Num - Num_S_Remainder 1571f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); 1572f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1573f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_GE_Den = (Remainder >= Den ? -1 : 0) 1574f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den, 1575f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 1576f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 157769239a98b69eefc70e68d82282b62583f6e92e10Vincent Lejeune ISD::SETUGE); 157869239a98b69eefc70e68d82282b62583f6e92e10Vincent Lejeune // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0) 157969239a98b69eefc70e68d82282b62583f6e92e10Vincent Lejeune SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num, 158069239a98b69eefc70e68d82282b62583f6e92e10Vincent Lejeune Num_S_Remainder, 1581f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 1582f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 158369239a98b69eefc70e68d82282b62583f6e92e10Vincent Lejeune ISD::SETUGE); 1584f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero 1585f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, 1586f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_GE_Zero); 1587f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1588f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Division result: 1589f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1590f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_A_One = Quotient + 1 1591f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, 1592f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 1593f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1594f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_S_One = Quotient - 1 1595f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, 1596f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 1597f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1598f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One) 1599f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 1600f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient, Quotient_A_One, ISD::SETEQ); 1601f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1602f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div) 1603f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 1604f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient_S_One, Div, ISD::SETEQ); 1605f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1606f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Rem result: 1607f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1608f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_S_Den = Remainder - Den 1609f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); 1610f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1611f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_A_Den = Remainder + Den 1612f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); 1613f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1614f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den) 1615f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 1616f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder, Remainder_S_Den, ISD::SETEQ); 1617f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1618f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem) 1619f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 1620f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_A_Den, Rem, ISD::SETEQ); 1621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { 1622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Div, 1623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Rem 1624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 1625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 1626f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1627f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1628cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op, 1629cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SelectionDAG &DAG) const { 1630cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1631cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 1632cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1633cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Zero = DAG.getConstant(0, VT); 1634cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue NegOne = DAG.getConstant(-1, VT); 1635cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1636cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS = Op.getOperand(0); 1637cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS = Op.getOperand(1); 1638cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1639cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); 1640cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); 1641cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); 1642cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RSign = LHSign; // Remainder sign is the same as LHS 1643cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1644cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); 1645cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); 1646cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1647cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); 1648cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); 1649cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1650cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); 1651cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Rem = Div.getValue(1); 1652cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1653cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); 1654cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); 1655cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1656cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); 1657cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); 1658cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1659cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Res[2] = { 1660cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Div, 1661cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Rem 1662cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines }; 1663cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getMergeValues(Res, DL); 1664cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1665cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1666cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const { 1667cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc SL(Op); 1668cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 1669cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1670cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // result = trunc(src) 1671cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // if (src > 0.0 && src != result) 1672cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // result += 1.0 1673cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1674cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 1675cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1676cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64); 1677cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue One = DAG.getConstantFP(1.0, MVT::f64); 1678cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1679cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 1680cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1681cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); 1682cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 1683cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 1684cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1685cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); 1686cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 1687cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1688cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1689cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const { 1690cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc SL(Op); 1691cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 1692cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1693cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines assert(Op.getValueType() == MVT::f64); 1694cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1695cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue Zero = DAG.getConstant(0, MVT::i32); 1696cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue One = DAG.getConstant(1, MVT::i32); 1697cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1698cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); 1699cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1700cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Extract the upper half, since this is where we will find the sign and 1701cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // exponent. 1702cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One); 1703cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1704cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const unsigned FractBits = 52; 1705cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const unsigned ExpBits = 11; 1706cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1707cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Extract the exponent. 1708cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_I32, SL, MVT::i32, 1709cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Hi, 1710cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(FractBits - 32, MVT::i32), 1711cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(ExpBits, MVT::i32)); 1712cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, 1713cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(1023, MVT::i32)); 1714cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1715cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Extract the sign bit. 1716cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, MVT::i32); 1717cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); 1718cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1719cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Extend back to to 64-bits. 1720cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue SignBit64 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, 1721cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Zero, SignBit); 1722cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); 1723cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1724cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); 1725cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue FractMask 1726cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines = DAG.getConstant((UINT64_C(1) << FractBits) - 1, MVT::i64); 1727cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1728cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); 1729cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); 1730cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); 1731cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1732cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32); 1733cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1734cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue FiftyOne = DAG.getConstant(FractBits - 1, MVT::i32); 1735cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1736cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); 1737cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); 1738cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1739cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); 1740cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); 1741cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1742cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); 1743cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1744cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1745cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { 1746cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc SL(Op); 1747cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 1748cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1749cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines assert(Op.getValueType() == MVT::f64); 1750cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1751cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines APFloat C1Val(APFloat::IEEEdouble, "0x1.0p+52"); 1752cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue C1 = DAG.getConstantFP(C1Val, MVT::f64); 1753cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); 1754cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1755cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); 1756cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); 1757cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1758cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); 1759cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1760cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines APFloat C2Val(APFloat::IEEEdouble, "0x1.fffffffffffffp+51"); 1761cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue C2 = DAG.getConstantFP(C2Val, MVT::f64); 1762cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1763cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 1764cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); 1765cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1766cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); 1767cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1768cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1769cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const { 1770cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // FNEARBYINT and FRINT are the same, except in their handling of FP 1771cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // exceptions. Those aren't really meaningful for us, and OpenCL only has 1772cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // rint, so just treat them as equivalent. 1773cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0)); 1774cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1775cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1776cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const { 1777cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc SL(Op); 1778cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 1779cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1780cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // result = trunc(src); 1781cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // if (src < 0.0 && src != result) 1782cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // result += -1.0. 1783cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1784cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 1785cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1786cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64); 1787cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const SDValue NegOne = DAG.getConstantFP(-1.0, MVT::f64); 1788cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1789cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 1790cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1791cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); 1792cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 1793cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 1794cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1795cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); 1796cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 1797cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1798cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1799aa1d078e7f42b605be03ff42d9b2e09923d3590dTom StellardSDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, 1800aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SelectionDAG &DAG) const { 1801aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDValue S0 = Op.getOperand(0); 1802aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDLoc DL(Op); 1803aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard if (Op.getValueType() != MVT::f32 || S0.getValueType() != MVT::i64) 1804aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard return SDValue(); 1805aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard 1806aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard // f32 uint_to_fp i64 1807aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, 1808aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard DAG.getConstant(0, MVT::i32)); 1809aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo); 1810aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, 1811aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard DAG.getConstant(1, MVT::i32)); 1812aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi); 1813aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi, 1814aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard DAG.getConstantFP(4294967296.0f, MVT::f32)); // 2^32 1815aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi); 1816aa1d078e7f42b605be03ff42d9b2e09923d3590dTom Stellard} 18174c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 181836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue AMDGPUTargetLowering::ExpandSIGN_EXTEND_INREG(SDValue Op, 181936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned BitsDiff, 182036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 182136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MVT VT = Op.getSimpleValueType(); 182236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 182336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Shift = DAG.getConstant(BitsDiff, VT); 182436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Shift left by 'Shift' bits. 182536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Op.getOperand(0), Shift); 182636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Signed shift Right by 'Shift' bits. 182736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(ISD::SRA, DL, VT, Shl, Shift); 182836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 182936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 183136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectionDAG &DAG) const { 183236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 183336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MVT VT = Op.getSimpleValueType(); 183436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MVT ScalarVT = VT.getScalarType(); 183536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 1837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 183836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Src = Op.getOperand(0); 1840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 184136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TODO: Don't scalarize on Evergreen? 1843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NElts = VT.getVectorNumElements(); 1844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> Args; 1845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.ExtractVectorElements(Src, Args, 0, NElts); 184636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType()); 1848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned I = 0; I < NElts; ++I) 1849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); 185036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Args); 1852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 185336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 1855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Custom DAG optimizations 1856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 185736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isU24(SDValue Op, SelectionDAG &DAG) { 1859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownZero, KnownOne; 1860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 1861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Op, KnownZero, KnownOne); 186236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24; 1864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 186536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isI24(SDValue Op, SelectionDAG &DAG) { 1867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 186836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // In order for this to be a signed 24-bit value, bit 23, must 1870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // be a sign bit. 1871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated 1872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // as unsigned 24-bit values. 1873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24; 1874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) { 1877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 1879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 1881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24); 1883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownZero, KnownOne; 1884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::TargetLoweringOpt TLO(DAG, true, true); 1885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) 1886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CommitTargetLoweringOpt(TLO); 1887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 188836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinestemplate <typename IntTy> 1890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, 1891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t Offset, uint32_t Width) { 1892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Width + Offset < 32) { 1893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IntTy Result = (Src0 << (32 - Offset - Width)) >> (32 - Width); 1894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getConstant(Result, MVT::i32); 189536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 189636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getConstant(Src0 >> Offset, MVT::i32); 1898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1900cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, 1901cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAGCombinerInfo &DCI) const { 1902cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = N->getValueType(0); 1903cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1904cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (VT.isVector() || VT.getSizeInBits() > 32) 1905cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(); 1906cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1907cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SelectionDAG &DAG = DCI.DAG; 1908cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(N); 1909cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1910cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue N0 = N->getOperand(0); 1911cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue N1 = N->getOperand(1); 1912cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Mul; 1913cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1914cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { 1915cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); 1916cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); 1917cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1); 1918cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { 1919cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); 1920cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); 1921cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1); 1922cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else { 1923cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return SDValue(); 1924cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 1925cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1926cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // We need to use sext even for MUL_U24, because MUL_U24 is used 1927cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // for signed multiply of 8 and 16-bit types. 1928cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getSExtOrTrunc(Mul, DL, VT); 1929cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1930cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, 1932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAGCombinerInfo &DCI) const { 1933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 1934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 1935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch(N->getOpcode()) { 1937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: break; 1938cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::MUL: 1939cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return performMulCombine(N, DCI); 1940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::MUL_I24: 1941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::MUL_U24: { 1942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 1943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N1 = N->getOperand(1); 1944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines simplifyI24(N0, DCI); 1945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines simplifyI24(N1, DCI); 1946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 1947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SELECT_CC: { 1949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CombineMinMax(N, DAG); 1950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_I32: 1952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_U32: { 1953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!N->getValueType(0).isVector() && 1954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Vector handling of BFE not implemented"); 1955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 1956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Width) 1957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t WidthVal = Width->getZExtValue() & 0x1f; 1960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (WidthVal == 0) 1961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getConstant(0, MVT::i32); 1962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Offset) 1965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BitsFrom = N->getOperand(0); 1968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t OffsetVal = Offset->getZExtValue() & 0x1f; 1969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32; 1971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (OffsetVal == 0) { 1973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is already sign / zero extended, so try to fold away extra BFEs. 1974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal); 1975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom); 1977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (OpSignBits >= SignBits) 1978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return BitsFrom; 1979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal); 1981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Signed) { 1982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is a sign_extend_inreg. Replace it to take advantage of existing 1983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // DAG Combines. If not eliminated, we will match back to BFE during 1984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // selection. 1985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TODO: The sext_inreg of extended types ends, although we can could 1987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // handle them in a single BFE. 1988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, 1989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getValueType(SmallVT)); 1990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1991dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT); 1993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *Val = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 1996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Signed) { 1997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return constantFoldBFE<int32_t>(DAG, 1998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val->getSExtValue(), 1999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OffsetVal, 2000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WidthVal); 2001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return constantFoldBFE<uint32_t>(DAG, 2004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val->getZExtValue(), 2005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OffsetVal, 2006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WidthVal); 2007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt Demanded = APInt::getBitsSet(32, 2010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OffsetVal, 2011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OffsetVal + WidthVal); 2012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((OffsetVal + WidthVal) >= 32) { 2014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShiftVal = DAG.getConstant(OffsetVal, MVT::i32); 2015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, 2016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BitsFrom, ShiftVal); 2017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownZero, KnownOne; 2020dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), 2021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !DCI.isBeforeLegalizeOps()); 2022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TLO.ShrinkDemandedConstant(BitsFrom, Demanded) || 2024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TLI.SimplifyDemandedBits(BitsFrom, Demanded, KnownZero, KnownOne, TLO)) { 2025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CommitTargetLoweringOpt(TLO); 2026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 203236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 203336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2034f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 2035f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Helper functions 2036f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 2037f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2038f95b1621887e3409ceec2db47e1b44271d934735Tom Stellardvoid AMDGPUTargetLowering::getOriginalFunctionArgs( 2039f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard SelectionDAG &DAG, 2040f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard const Function *F, 2041f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard const SmallVectorImpl<ISD::InputArg> &Ins, 2042f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard SmallVectorImpl<ISD::InputArg> &OrigIns) const { 2043f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 2044f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 2045f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard if (Ins[i].ArgVT == Ins[i].VT) { 2046f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard OrigIns.push_back(Ins[i]); 2047f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard continue; 2048f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard } 2049f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 2050f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard EVT VT; 2051f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard if (Ins[i].ArgVT.isVector() && !Ins[i].VT.isVector()) { 2052f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard // Vector has been split into scalars. 2053f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard VT = Ins[i].ArgVT.getVectorElementType(); 2054f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard } else if (Ins[i].VT.isVector() && Ins[i].ArgVT.isVector() && 2055f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard Ins[i].ArgVT.getVectorElementType() != 2056f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard Ins[i].VT.getVectorElementType()) { 2057f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard // Vector elements have been promoted 2058f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard VT = Ins[i].ArgVT; 2059f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard } else { 2060f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard // Vector has been spilt into smaller vectors. 2061f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard VT = Ins[i].VT; 2062f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard } 2063f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 2064f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard ISD::InputArg Arg(Ins[i].Flags, VT, VT, Ins[i].Used, 2065f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard Ins[i].OrigArgIndex, Ins[i].PartOffset); 2066f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard OrigIns.push_back(Arg); 2067f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard } 2068f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard} 2069f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 2070f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const { 2071f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 2072f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->isExactlyValue(1.0); 2073f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2074f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 2075f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isAllOnesValue(); 2076f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2077f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 2078f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 2079f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2080f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const { 2081f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 2082f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->getValueAPF().isZero(); 2083f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2084f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 2085f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isNullValue(); 2086f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2087f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 2088f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 2089f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2090f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 2091f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const TargetRegisterClass *RC, 2092f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned Reg, EVT VT) const { 2093f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 2094f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineRegisterInfo &MRI = MF.getRegInfo(); 2095f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned VirtualRegister; 2096f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (!MRI.isLiveIn(Reg)) { 2097f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.createVirtualRegister(RC); 2098f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MRI.addLiveIn(Reg, VirtualRegister); 2099f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else { 2100f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.getLiveInVirtReg(Reg); 2101f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2102f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getRegister(VirtualRegister, VT); 2103f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 2104f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2105f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 2106f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2107f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardconst char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { 2108f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Opcode) { 2109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: return nullptr; 2110f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDIL DAG nodes 2111f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(CALL); 2112f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMUL); 2113f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(RET_FLAG); 2114f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(BRANCH_COND); 2115f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 2116f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDGPU DAG nodes 2117f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(DWORDADDR) 2118f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FRACT) 2119cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(CLAMP) 2120f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMAX) 2121f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMAX) 2122f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMAX) 2123f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMIN) 2124f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMIN) 2125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMIN) 2126cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(URECIP) 2127cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(DIV_SCALE) 2128cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(DIV_FMAS) 2129cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(DIV_FIXUP) 2130cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(TRIG_PREOP) 2131cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(RCP) 2132cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(RSQ) 2133cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(RSQ_LEGACY) 2134cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(RSQ_CLAMPED) 2135cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(DOT4) 213636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NODE_NAME_CASE(BFE_U32) 213736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NODE_NAME_CASE(BFE_I32) 213836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NODE_NAME_CASE(BFI) 213936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NODE_NAME_CASE(BFM) 2140cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(BREV) 2141dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NODE_NAME_CASE(MUL_U24) 2142dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NODE_NAME_CASE(MUL_I24) 2143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NODE_NAME_CASE(MAD_U24) 2144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NODE_NAME_CASE(MAD_I24) 2145f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(EXPORT) 2146c7e1888d93f4cb2982266986f3af7e99df631fa1Tom Stellard NODE_NAME_CASE(CONST_ADDRESS) 2147c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_LOAD) 2148c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_STORE) 214968db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_CONSTANT) 215068db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_INPUT) 215168db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLE) 215268db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEB) 215368db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLED) 215468db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEL) 2155cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(CVT_F32_UBYTE0) 2156cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(CVT_F32_UBYTE1) 2157cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(CVT_F32_UBYTE2) 2158cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(CVT_F32_UBYTE3) 2159cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NODE_NAME_CASE(BUILD_VERTICAL_VECTOR) 2160ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard NODE_NAME_CASE(STORE_MSKOR) 2161a3c2bcf0ee2f63584f7a1e9df9fa153a8b5dfea1Tom Stellard NODE_NAME_CASE(TBUFFER_STORE_FORMAT) 2162f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2163f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 216436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic void computeKnownBitsForMinMax(const SDValue Op0, 2166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue Op1, 2167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt &KnownZero, 2168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt &KnownOne, 2169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SelectionDAG &DAG, 2170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Depth) { 217136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines APInt Op0Zero, Op0One; 217236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines APInt Op1Zero, Op1One; 2173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Op0, Op0Zero, Op0One, Depth); 2174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Op1, Op1Zero, Op1One, Depth); 217536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 217636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines KnownZero = Op0Zero & Op1Zero; 217736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines KnownOne = Op0One & Op1One; 217836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 217936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AMDGPUTargetLowering::computeKnownBitsForTargetNode( 218136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SDValue Op, 218236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines APInt &KnownZero, 218336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines APInt &KnownOne, 218436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SelectionDAG &DAG, 218536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) const { 218636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 218736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything. 2188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownZero2; 2190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownOne2; 219136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Opc = Op.getOpcode(); 2192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 219336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (Opc) { 2194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2195dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 219636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::INTRINSIC_WO_CHAIN: { 219736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // FIXME: The intrinsic should just use the node. 219836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 219936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_imax: 220036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_umax: 220136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_imin: 220236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUIntrinsic::AMDGPU_umin: 2203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines computeKnownBitsForMinMax(Op.getOperand(1), Op.getOperand(2), 2204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero, KnownOne, DAG, Depth); 220536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 220636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 220736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 220836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 220936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 221036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 221136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 221236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUISD::SMAX: 221336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUISD::UMAX: 221436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUISD::SMIN: 221536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case AMDGPUISD::UMIN: 2216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines computeKnownBitsForMinMax(Op.getOperand(0), Op.getOperand(1), 2217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero, KnownOne, DAG, Depth); 221836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 2219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_I32: 2221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_U32: { 2222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CWidth) 2224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 2225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BitWidth = 32; 2227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t Width = CWidth->getZExtValue() & 0x1f; 2228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Width == 0) { 2229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero = APInt::getAllOnesValue(BitWidth); 2230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownOne = APInt::getNullValue(BitWidth); 2231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 2232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This could do a lot more. If offset is 0, should be the same as 2235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // sign_extend_inreg implementation, but that involves duplicating it. 2236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Opc == AMDGPUISD::BFE_I32) 2237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownOne = APInt::getHighBitsSet(BitWidth, BitWidth - Width); 2238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 2239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - Width); 2240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 224136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 224236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 2243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 2245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesunsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode( 2247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Op, 2248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SelectionDAG &DAG, 2249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Depth) const { 2250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 2251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_I32: { 2252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Width) 2254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return 1; 2255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SignBits = 32 - Width->getZExtValue() + 1; 2257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 2258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Offset || !Offset->isNullValue()) 2259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SignBits; 2260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TODO: Could probably figure something out with non-0 offsets. 2262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); 2263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::max(SignBits, Op0SignBits); 2264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AMDGPUISD::BFE_U32: { 2267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1; 2269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return 1; 2273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 227436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 2275