AMDGPUISelLowering.cpp revision d08a9303614355cfdcac5f2c27c09ce809565423
1f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 3f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// The LLVM Compiler Infrastructure 4f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 5f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// This file is distributed under the University of Illinois Open Source 6f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// License. See LICENSE.TXT for details. 7f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 8f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 9f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 10f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \file 11f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief This is the parent TargetLowering class for hardware code gen 12f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// targets. 13f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 14f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 15f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 16f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "AMDGPUISelLowering.h" 17e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "AMDGPU.h" 1890c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPURegisterInfo.h" 1990c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUSubtarget.h" 205c35290fa35ae234fed02496404cb0fc37e1c8a5Benjamin Kramer#include "AMDILIntrinsicInfo.h" 21f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard#include "R600MachineFunctionInfo.h" 22e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "SIMachineFunctionInfo.h" 2390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "llvm/CodeGen/CallingConvLower.h" 24f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineFunction.h" 25f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 26f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 27f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 28e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard#include "llvm/IR/DataLayout.h" 29f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 30f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardusing namespace llvm; 31f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 3290c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUGenCallingConv.inc" 3390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 34f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardAMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) : 35f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TargetLowering(TM, new TargetLoweringObjectFileELF()) { 36f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 37f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Initialize target lowering borrowed from AMDIL 38f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard InitAMDILLowering(); 39f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 40f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // We need to custom lower some of the intrinsics 41f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 42f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 43f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Library functions. These default to Expand, but we have instructions 44f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // for them. 45f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FCEIL, MVT::f32, Legal); 46f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FEXP2, MVT::f32, Legal); 47f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FPOW, MVT::f32, Legal); 48f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FLOG2, MVT::f32, Legal); 49f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FABS, MVT::f32, Legal); 50f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 51f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FRINT, MVT::f32, Legal); 52f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 53ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard // The hardware supports ROTR, but not ROTL 54ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard setOperationAction(ISD::ROTL, MVT::i32, Expand); 55ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard 56f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Lower floating point store/load to integer store/load to reduce the number 57f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // of patterns in tablegen. 58f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::f32, Promote); 59f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); 60f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 61fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard setOperationAction(ISD::STORE, MVT::v2f32, Promote); 62fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); 63fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard 64f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::v4f32, Promote); 65f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); 66f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 6768e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::STORE, MVT::f64, Promote); 6868e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64); 6968e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 707a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // Custom lowering of vector stores is required for local address space 717a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // stores. 727a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard setOperationAction(ISD::STORE, MVT::v4i32, Custom); 737a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // XXX: Native v2i32 local address space stores are possible, but not 747a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // currently implemented. 757a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard setOperationAction(ISD::STORE, MVT::v2i32, Custom); 767a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 774c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); 784c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); 794c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); 804c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard // XXX: This can be change to Custom, once ExpandVectorStores can 814c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard // handle 64-bit stores. 824c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); 834c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 84f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::f32, Promote); 85f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); 86f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 87ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 88ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); 89ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard 90f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::v4f32, Promote); 91f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); 92f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 9368e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::LOAD, MVT::f64, Promote); 9468e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64); 9568e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 96a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 97a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); 98a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); 99a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); 100692ee102ebef535d311c35d53457028083e5c5beTom Stellard 10130d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v2i8, Expand); 10230d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v2i8, Expand); 10330d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i8, Expand); 10430d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v4i8, Expand); 10530d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v4i8, Expand); 10630d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i8, Expand); 10730d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, Expand); 10830d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, Expand); 10930d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, Expand); 11030d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, Expand); 11130d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, Expand); 11230d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, Expand); 11330d84d8dfa0433088d541c66b92af0da3855bc9cTom Stellard 114d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard setOperationAction(ISD::FNEG, MVT::v2f32, Expand); 115d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard setOperationAction(ISD::FNEG, MVT::v4f32, Expand); 116d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard 11745b14e341a8a85e877d001bbd43f5e2b25b61cb8Christian Konig setOperationAction(ISD::MUL, MVT::i64, Expand); 11845b14e341a8a85e877d001bbd43f5e2b25b61cb8Christian Konig 119f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UDIV, MVT::i32, Expand); 120f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 121f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UREM, MVT::i32, Expand); 122f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, MVT::v2f32, Expand); 123f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, MVT::v4f32, Expand); 124f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 1255464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard static const MVT::SimpleValueType IntTypes[] = { 1265464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::v2i32, MVT::v4i32 127f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry }; 1280991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard const size_t NumIntTypes = array_lengthof(IntTypes); 129f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 1300991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard for (unsigned int x = 0; x < NumIntTypes; ++x) { 1315464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::SimpleValueType VT = IntTypes[x]; 132f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry //Expand the following operations for the current type by default 133f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::ADD, VT, Expand); 134f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::AND, VT, Expand); 135e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_SINT, VT, Expand); 136e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_UINT, VT, Expand); 137f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::MUL, VT, Expand); 138f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::OR, VT, Expand); 139f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SHL, VT, Expand); 140e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::SINT_TO_FP, VT, Expand); 141f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SRL, VT, Expand); 142f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SRA, VT, Expand); 143f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SUB, VT, Expand); 144f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::UDIV, VT, Expand); 145e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::UINT_TO_FP, VT, Expand); 146f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::UREM, VT, Expand); 147f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, VT, Expand); 148f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::XOR, VT, Expand); 149f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry } 1500991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard 1515464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard static const MVT::SimpleValueType FloatTypes[] = { 1525464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::v2f32, MVT::v4f32 1530991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard }; 1540991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard const size_t NumFloatTypes = array_lengthof(FloatTypes); 1550991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard 1560991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard for (unsigned int x = 0; x < NumFloatTypes; ++x) { 1575464a92861c76f1e091cd219dee71ce9858eb195Tom Stellard MVT::SimpleValueType VT = FloatTypes[x]; 1580991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FADD, VT, Expand); 1590991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FDIV, VT, Expand); 16084c0bd9803547b1cb09a20032bbc151a718b9457Tom Stellard setOperationAction(ISD::FFLOOR, VT, Expand); 1610991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FMUL, VT, Expand); 1623cae823f69f083c5eef19f0ea3128c48d7807c9bTom Stellard setOperationAction(ISD::FRINT, VT, Expand); 1630991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard setOperationAction(ISD::FSUB, VT, Expand); 1640991c314d7c1a2052963dc89af1d2f07134488b6Tom Stellard } 165f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 166f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1672b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 1682b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard// Target Information 1692b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 1702b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 1712b272a1c8cb6d9f02223a598495d84cd9d75b13dTom StellardMVT AMDGPUTargetLowering::getVectorIdxTy() const { 1722b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard return MVT::i32; 1732b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard} 1742b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 1752b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 176f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 1771f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard// Target Properties 1781f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 1791f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1801f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { 1811f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 1821f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 1831f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 1841f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1851f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFNegFree(EVT VT) const { 1861f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 1871f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 1881f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 1891f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1901f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 191f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// TargetLowering Callbacks 192f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 193f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 19490c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konigvoid AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, 19590c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig const SmallVectorImpl<ISD::InputArg> &Ins) const { 19690c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 19790c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig State.AnalyzeFormalArguments(Ins, CC_AMDGPU); 198f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 199f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 200f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerReturn( 201f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain, 202f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CallingConv::ID CallConv, 203f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard bool isVarArg, 204f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<ISD::OutputArg> &Outs, 205f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<SDValue> &OutVals, 206ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, SelectionDAG &DAG) const { 207f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); 208f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 209f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 210f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 211f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Target specific lowering 212f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 213f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 214f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) 215f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const { 216f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Op.getOpcode()) { 217f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: 218f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getNode()->dump(); 219f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Custom lowering code for this" 220f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard "instruction is not implemented yet!"); 221f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 222f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDIL DAG lowering 223f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SDIV: return LowerSDIV(Op, DAG); 224f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SREM: return LowerSREM(Op, DAG); 225f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 226f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::BRCOND: return LowerBRCOND(Op, DAG); 227f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDGPU DAG lowering 228a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 229a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 230f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 2317a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard case ISD::STORE: return LowerSTORE(Op, DAG); 232f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 233f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 234f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Op; 235f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 236f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 237e3d4cbc7d25061441adafa47450a31571c87bf85Tom StellardSDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, 238e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SDValue Op, 239e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SelectionDAG &DAG) const { 240e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 241e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard const DataLayout *TD = getTargetMachine().getDataLayout(); 242e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op); 243da25cd3e6de8f21005590c2de49868f883cf2410Tom Stellard 244da25cd3e6de8f21005590c2de49868f883cf2410Tom Stellard assert(G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS); 245e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard // XXX: What does the value of G->getOffset() mean? 246e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard assert(G->getOffset() == 0 && 247e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard "Do not know what to do with an non-zero offset"); 248e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 249e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard unsigned Offset = MFI->LDSSize; 250e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard const GlobalValue *GV = G->getGlobal(); 251e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType()); 252e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 253e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard // XXX: Account for alignment? 254e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard MFI->LDSSize += Size; 255e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 256da25cd3e6de8f21005590c2de49868f883cf2410Tom Stellard return DAG.getConstant(Offset, getPointerTy(G->getAddressSpace())); 257e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard} 258e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 259a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellardvoid AMDGPUTargetLowering::ExtractVectorElements(SDValue Op, SelectionDAG &DAG, 260a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVectorImpl<SDValue> &Args, 261a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Start, 262a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Count) const { 263a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard EVT VT = Op.getValueType(); 264a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard for (unsigned i = Start, e = Start + Count; i != e; ++i) { 265a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), 266a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard VT.getVectorElementType(), 267a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard Op, DAG.getConstant(i, MVT::i32))); 268a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard } 269a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 270a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 271a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op, 272a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 273a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 274a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue A = Op.getOperand(0); 275a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue B = Op.getOperand(1); 276a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 277a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(A, DAG, Args, 0, 278a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard A.getValueType().getVectorNumElements()); 279a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(B, DAG, Args, 0, 280a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard B.getValueType().getVectorNumElements()); 281a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 282a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), 283a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard &Args[0], Args.size()); 284a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 285a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 286a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 287a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 288a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 289a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 290a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard EVT VT = Op.getValueType(); 291a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 292a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(Op.getOperand(0), DAG, Args, Start, 293a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard VT.getVectorNumElements()); 294a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 295a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), 296a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard &Args[0], Args.size()); 297a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 298a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 299a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 300f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 301f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 302f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 303ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 304f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 305f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 306f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (IntrinsicID) { 307f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return Op; 308f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_abs: 309f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicIABS(Op, DAG); 310f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_exp: 311f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1)); 312f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_lrp: 313f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicLRP(Op, DAG); 314f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_fraction: 315f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 316f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_max: 317f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, Op.getOperand(1), 318f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 319f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imax: 320f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1), 321f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 322f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umax: 323f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1), 324f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 325f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_min: 326f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, Op.getOperand(1), 327f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 328f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imin: 329f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1), 330f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 331f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umin: 332f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1), 333f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 334f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_round_nearest: 335f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1)); 336f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 337f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 338f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 339f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard///IABS(a) = SMAX(sub(0, a), a) 340f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op, 341f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 342f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 343ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 344f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 345f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 346f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 347f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 348f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1)); 349f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 350f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 351f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// Linear Interpolation 352f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// LRP(a, b, c) = muladd(a, b, (1 - a) * c) 353f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op, 354f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 355ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 356f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 357f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT, 358f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstantFP(1.0f, MVT::f32), 359f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 360f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA, 361f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(3)); 362e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune return DAG.getNode(ISD::FADD, DL, VT, 363e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)), 364e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune OneSubAC); 365f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 366f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 367f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief Generate Min/Max node 368f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerMinMax(SDValue Op, 369f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 370ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 371f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 372f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 373f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue LHS = Op.getOperand(0); 374f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RHS = Op.getOperand(1); 375f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue True = Op.getOperand(2); 376f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue False = Op.getOperand(3); 377f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue CC = Op.getOperand(4); 378f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 379f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (VT != MVT::f32 || 380f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard !((LHS == True && RHS == False) || (LHS == False && RHS == True))) { 381f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return SDValue(); 382f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 383f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 384f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 385f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (CCOpcode) { 386f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOEQ: 387f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETONE: 388f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUNE: 389f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETNE: 390f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUEQ: 391f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETEQ: 392f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE: 393f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE2: 394f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE: 395f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE2: 396f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUO: 397f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETO: 398f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Operation should already be optimised !"); 399f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULE: 400f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULT: 401f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLE: 402f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLT: 403f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLE: 404f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLT: { 405f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (LHS == True) 406f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS); 407f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else 408f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS); 409f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 410f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGT: 411f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGE: 412f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGE: 413f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGE: 414f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGT: 415f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGT: { 416f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (LHS == True) 417f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS); 418f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else 419f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS); 420f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 421f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETCC_INVALID: 422f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Invalid setcc condcode !"); 423f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 424f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Op; 425f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 426f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 427d08a9303614355cfdcac5f2c27c09ce809565423Tom StellardSDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue &Op, 428d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SelectionDAG &DAG) const { 429d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard LoadSDNode *Load = dyn_cast<LoadSDNode>(Op); 430d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT MemEltVT = Load->getMemoryVT().getVectorElementType(); 431d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT EltVT = Op.getValueType().getVectorElementType(); 432d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard EVT PtrVT = Load->getBasePtr().getValueType(); 433d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard unsigned NumElts = Load->getMemoryVT().getVectorNumElements(); 434d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SmallVector<SDValue, 8> Loads; 435d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDLoc SL(Op); 436d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 437d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard for (unsigned i = 0, e = NumElts; i != e; ++i) { 438d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(), 439d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), PtrVT)); 440d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Loads.push_back(DAG.getExtLoad(Load->getExtensionType(), SL, EltVT, 441d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Load->getChain(), Ptr, 442d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard MachinePointerInfo(Load->getMemOperand()->getValue()), 443d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard MemEltVT, Load->isVolatile(), Load->isNonTemporal(), 444d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Load->getAlignment())); 445d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard } 446d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard return DAG.getNode(ISD::BUILD_VECTOR, SL, Op.getValueType(), &Loads[0], 447d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Loads.size()); 448d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard} 449d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 4507a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op, 4517a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SelectionDAG &DAG) const { 4527a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard StoreSDNode *Store = dyn_cast<StoreSDNode>(Op); 4537a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemVT = Store->getMemoryVT(); 4547a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemBits = MemVT.getSizeInBits(); 4557a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 4567a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // Byte stores are really expensive, so if possible, try to pack 4577a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // 32-bit vector truncatating store into an i32 store. 4587a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard // XXX: We could also handle optimize other vector bitwidths 4597a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (!MemVT.isVector() || MemBits > 32) { 4607a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SDValue(); 4617a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 4627a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 4637a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDLoc DL(Op); 4647a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard const SDValue &Value = Store->getValue(); 4657a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT VT = Value.getValueType(); 4667a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard const SDValue &Ptr = Store->getBasePtr(); 4677a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemEltVT = MemVT.getVectorElementType(); 4687a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemEltBits = MemEltVT.getSizeInBits(); 4697a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned MemNumElements = MemVT.getVectorNumElements(); 4707a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()); 4717a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Mask; 4727a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard switch(MemEltBits) { 4737a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard case 8: 4747a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Mask = DAG.getConstant(0xFF, PackedVT); 4757a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard break; 4767a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard case 16: 4777a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Mask = DAG.getConstant(0xFFFF, PackedVT); 4787a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard break; 4797a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard default: 4807a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard llvm_unreachable("Cannot lower this vector store"); 4817a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 4827a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue PackedValue; 4837a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard for (unsigned i = 0; i < MemNumElements; ++i) { 4847a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT ElemVT = VT.getVectorElementType(); 4857a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value, 4867a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard DAG.getConstant(i, MVT::i32)); 4877a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Elt = DAG.getZExtOrTrunc(Elt, DL, PackedVT); 4887a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Elt = DAG.getNode(ISD::AND, DL, PackedVT, Elt, Mask); 4897a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Shift = DAG.getConstant(MemEltBits * i, PackedVT); 4907a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Elt = DAG.getNode(ISD::SHL, DL, PackedVT, Elt, Shift); 4917a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (i == 0) { 4927a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard PackedValue = Elt; 4937a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } else { 4947a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard PackedValue = DAG.getNode(ISD::OR, DL, PackedVT, PackedValue, Elt); 4957a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 4967a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 4977a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr, 4987a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard MachinePointerInfo(Store->getMemOperand()->getValue()), 4997a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->isVolatile(), Store->isNonTemporal(), 5007a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getAlignment()); 5017a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 5027a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 5037a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op, 5047a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SelectionDAG &DAG) const { 5057a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard StoreSDNode *Store = cast<StoreSDNode>(Op); 5067a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT MemEltVT = Store->getMemoryVT().getVectorElementType(); 5077a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT EltVT = Store->getValue().getValueType().getVectorElementType(); 5087a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard EVT PtrVT = Store->getBasePtr().getValueType(); 5097a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard unsigned NumElts = Store->getMemoryVT().getVectorNumElements(); 5107a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDLoc SL(Op); 5117a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 5127a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SmallVector<SDValue, 8> Chains; 5137a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 5147a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard for (unsigned i = 0, e = NumElts; i != e; ++i) { 5157a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 5167a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getValue(), DAG.getConstant(i, MVT::i32)); 5177a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, 5187a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getBasePtr(), 5197a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), 5207a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard PtrVT)); 5218e78012457682d335ee97cf2859dfe03b7e2ae93Tom Stellard Chains.push_back(DAG.getTruncStore(Store->getChain(), SL, Val, Ptr, 5227a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard MachinePointerInfo(Store->getMemOperand()->getValue()), 5238e78012457682d335ee97cf2859dfe03b7e2ae93Tom Stellard MemEltVT, Store->isVolatile(), Store->isNonTemporal(), 5247a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getAlignment())); 5257a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 5267a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, &Chains[0], NumElts); 5277a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 5287a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard 5297a0282daeb214f14d75249cc2d90302c44586c4eTom StellardSDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 5307a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG); 5317a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (Result.getNode()) { 5327a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return Result; 5337a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 534f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 5357a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard StoreSDNode *Store = cast<StoreSDNode>(Op); 5367a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard if (Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && 5377a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard Store->getValue().getValueType().isVector()) { 5387a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SplitVectorStore(Op, DAG); 5397a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard } 5407a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard return SDValue(); 5417a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard} 542f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 543f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 544f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 545ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 546f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 547f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 548f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num = Op.getOperand(0); 549f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Den = Op.getOperand(1); 550f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 551f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVector<SDValue, 8> Results; 552f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 553f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP = URECIP(Den) = 2^32 / Den + e 554f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // e is rounding error. 555f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); 556f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 557f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_LO = umulo(RCP, Den) */ 558f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den); 559f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 560f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_HI = mulhu (RCP, Den) */ 561f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); 562f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 563f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // NEG_RCP_LO = -RCP_LO 564f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 565f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_LO); 566f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 567f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO) 568f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 569f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NEG_RCP_LO, RCP_LO, 570f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 571f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate the rounding error from the URECIP instruction 572f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // E = mulhu(ABS_RCP_LO, RCP) 573f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); 574f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 575f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_A_E = RCP + E 576f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); 577f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 578f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_S_E = RCP - E 579f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); 580f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 581f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E) 582f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 583f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_A_E, RCP_S_E, 584f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 585f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient = mulhu(Tmp0, Num) 586f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); 587f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 588f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Num_S_Remainder = Quotient * Den 589f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den); 590f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 591f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder = Num - Num_S_Remainder 592f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); 593f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 594f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_GE_Den = (Remainder >= Den ? -1 : 0) 595f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den, 596f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 597f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 598f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETGE); 599f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_GE_Zero = (Remainder >= 0 ? -1 : 0) 600f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Remainder, 601f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 602f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 603f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 604f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETGE); 605f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero 606f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, 607f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_GE_Zero); 608f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 609f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Division result: 610f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 611f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_A_One = Quotient + 1 612f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, 613f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 614f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 615f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_S_One = Quotient - 1 616f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, 617f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 618f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 619f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One) 620f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 621f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient, Quotient_A_One, ISD::SETEQ); 622f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 623f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div) 624f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 625f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient_S_One, Div, ISD::SETEQ); 626f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 627f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Rem result: 628f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 629f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_S_Den = Remainder - Den 630f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); 631f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 632f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_A_Den = Remainder + Den 633f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); 634f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 635f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den) 636f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 637f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder, Remainder_S_Den, ISD::SETEQ); 638f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 639f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem) 640f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 641f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_A_Den, Rem, ISD::SETEQ); 642f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Ops[2]; 643f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Ops[0] = Div; 644f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Ops[1] = Rem; 645f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getMergeValues(Ops, 2, DL); 646f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 647f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 6484c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 649f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 650f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Helper functions 651f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 652f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 653f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const { 654f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 655f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->isExactlyValue(1.0); 656f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 657f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 658f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isAllOnesValue(); 659f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 660f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 661f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 662f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 663f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const { 664f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 665f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->getValueAPF().isZero(); 666f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 667f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 668f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isNullValue(); 669f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 670f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 671f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 672f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 673f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 674f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const TargetRegisterClass *RC, 675f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned Reg, EVT VT) const { 676f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 677f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineRegisterInfo &MRI = MF.getRegInfo(); 678f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned VirtualRegister; 679f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (!MRI.isLiveIn(Reg)) { 680f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.createVirtualRegister(RC); 681f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MRI.addLiveIn(Reg, VirtualRegister); 682f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else { 683f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.getLiveInVirtReg(Reg); 684f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 685f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getRegister(VirtualRegister, VT); 686f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 687f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 688f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 689f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 690f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardconst char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { 691f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Opcode) { 692f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return 0; 693f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDIL DAG nodes 694f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(CALL); 695f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMUL); 696f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(DIV_INF); 697f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(RET_FLAG); 698f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(BRANCH_COND); 699f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 700f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDGPU DAG nodes 701f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(DWORDADDR) 702f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FRACT) 703f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMAX) 704f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMAX) 705f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMAX) 706f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMIN) 707f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMIN) 708f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMIN) 709f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(URECIP) 710f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(EXPORT) 711c7e1888d93f4cb2982266986f3af7e99df631fa1Tom Stellard NODE_NAME_CASE(CONST_ADDRESS) 712c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_LOAD) 713c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_STORE) 71468db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_CONSTANT) 71568db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_INPUT) 71668db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLE) 71768db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEB) 71868db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLED) 71968db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEL) 720ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard NODE_NAME_CASE(STORE_MSKOR) 721f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 722f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 723