1f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===// 2f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 3f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// The LLVM Compiler Infrastructure 4f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 5f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// This file is distributed under the University of Illinois Open Source 6f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// License. See LICENSE.TXT for details. 7f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 8f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 9f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 10f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \file 11f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief Custom DAG lowering for R600 12f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 13f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 14f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 15f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600ISelLowering.h" 16cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines#include "AMDGPUFrameLowering.h" 17cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines#include "AMDGPUIntrinsicInfo.h" 18cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines#include "AMDGPUSubtarget.h" 19f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600Defines.h" 20f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600InstrInfo.h" 21f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600MachineFunctionInfo.h" 22f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard#include "llvm/CodeGen/CallingConvLower.h" 23c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard#include "llvm/CodeGen/MachineFrameInfo.h" 24f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineInstrBuilder.h" 25f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 26f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 270b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Argument.h" 280b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 29f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 30f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardusing namespace llvm; 31f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 32f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardR600TargetLowering::R600TargetLowering(TargetMachine &TM) : 33c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune AMDGPUTargetLowering(TM), 34c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune Gen(TM.getSubtarget<AMDGPUSubtarget>().getGeneration()) { 35f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::v4f32, &AMDGPU::R600_Reg128RegClass); 36f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::f32, &AMDGPU::R600_Reg32RegClass); 37f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::v4i32, &AMDGPU::R600_Reg128RegClass); 38f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::i32, &AMDGPU::R600_Reg32RegClass); 39692ee102ebef535d311c35d53457028083e5c5beTom Stellard addRegisterClass(MVT::v2f32, &AMDGPU::R600_Reg64RegClass); 40692ee102ebef535d311c35d53457028083e5c5beTom Stellard addRegisterClass(MVT::v2i32, &AMDGPU::R600_Reg64RegClass); 41692ee102ebef535d311c35d53457028083e5c5beTom Stellard 42f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard computeRegisterProperties(); 43f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 449c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard // Set condition code actions 459c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETO, MVT::f32, Expand); 469c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 4712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLT, MVT::f32, Expand); 489c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETLE, MVT::f32, Expand); 4912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETOLT, MVT::f32, Expand); 5012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 519c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 529c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 539c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); 549c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 5512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 5612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULE, MVT::f32, Expand); 5712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 5812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLE, MVT::i32, Expand); 5912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLT, MVT::i32, Expand); 6012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULE, MVT::i32, Expand); 6112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULT, MVT::i32, Expand); 6212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 63c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune setOperationAction(ISD::FCOS, MVT::f32, Custom); 64c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune setOperationAction(ISD::FSIN, MVT::f32, Custom); 65c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune 66f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SETCC, MVT::v4i32, Expand); 67692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::SETCC, MVT::v2i32, Expand); 68f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 69d41650590669bf561d8f3bcae1204f11354954dcTom Stellard setOperationAction(ISD::BR_CC, MVT::i32, Expand); 70d41650590669bf561d8f3bcae1204f11354954dcTom Stellard setOperationAction(ISD::BR_CC, MVT::f32, Expand); 71cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::BRCOND, MVT::Other, Custom); 72f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 73f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FSUB, MVT::f32, Expand); 74f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 75f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 76f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 77f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i1, Custom); 78f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 79f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 80f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 81f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 829c6b0b0ccebf9d9bf0f357a1c72ef941c5bbb2b2Tom Stellard setOperationAction(ISD::SETCC, MVT::i32, Expand); 839c6b0b0ccebf9d9bf0f357a1c72ef941c5bbb2b2Tom Stellard setOperationAction(ISD::SETCC, MVT::f32, Expand); 84f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom); 85f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 86756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::i32, Expand); 87756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::f32, Expand); 88756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 89756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::v4i32, Expand); 90dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 91dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Expand sign extension of vectors 92dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 93dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 94dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 95dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand); 96dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand); 97dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 98dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 99dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 100dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand); 101dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand); 102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand); 106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand); 107dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand); 110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand); 111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand); 113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 114f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 115c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Legalize loads and stores to the private address space. 116c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::LOAD, MVT::i32, Custom); 117692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 118c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 11929f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault 12029f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address 12129f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault // spaces, so it is custom lowered to handle those where it isn't. 122a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::i8, Custom); 123a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard setLoadExtAction(ISD::SEXTLOAD, MVT::i16, Custom); 124a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::i8, Custom); 125a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard setLoadExtAction(ISD::ZEXTLOAD, MVT::i16, Custom); 12629f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault setLoadExtAction(ISD::EXTLOAD, MVT::i8, Custom); 12729f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault setLoadExtAction(ISD::EXTLOAD, MVT::i16, Custom); 12829f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault 129c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::STORE, MVT::i8, Custom); 130f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::i32, Custom); 131692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::STORE, MVT::v2i32, Custom); 132f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::v4i32, Custom); 133ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard setTruncStoreAction(MVT::i32, MVT::i8, Custom); 134ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard setTruncStoreAction(MVT::i32, MVT::i16, Custom); 135f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1369f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setOperationAction(ISD::LOAD, MVT::i32, Custom); 1379f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 138c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 139c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 140cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom); 141cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom); 142cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 143cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 144cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 145cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom); 146cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom); 147cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 148cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 149cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 150f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setTargetDAGCombine(ISD::FP_ROUND); 1511234c9be42b4ebd4b398df461123205dccf3706cTom Stellard setTargetDAGCombine(ISD::FP_TO_SINT); 1529f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 1531234c9be42b4ebd4b398df461123205dccf3706cTom Stellard setTargetDAGCombine(ISD::SELECT_CC); 15415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 155f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 156cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SUB, MVT::i64, Expand); 157cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // These should be replaced by UDVIREM, but it does not happen automatically 159dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // during Type Legalization 160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UDIV, MVT::i64, Custom); 161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UREM, MVT::i64, Custom); 162cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SDIV, MVT::i64, Custom); 163cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SREM, MVT::i64, Custom); 164cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 165cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 166cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // to be Legal/Custom in order to avoid library calls. 167cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 168cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 169cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 171a3e39dc7055486cbf514ccd868cfabc69d7f6f4eMichel Danzer setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 172a3e39dc7055486cbf514ccd868cfabc69d7f6f4eMichel Danzer 173cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 174cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (MVT VT : ScalarIntVTs) { 175cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ADDC, VT, Expand); 176cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SUBC, VT, Expand); 177cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::ADDE, VT, Expand); 178cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines setOperationAction(ISD::SUBE, VT, Expand); 179cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 180cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 181ff1ccdf9c58d12b142b3ab7473ac531cc5728a06Tom Stellard setBooleanContents(ZeroOrNegativeOneBooleanContent); 182aa6deab60871d251bf121c2b894105e68f349ba5Tom Stellard setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 183f15dfe4eb48e8e2ff02a30bc8ba9112108f9b83dTom Stellard setSchedulingPreference(Sched::Source); 184f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 185f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 186f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardMachineBasicBlock * R600TargetLowering::EmitInstrWithCustomInserter( 187f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr * MI, MachineBasicBlock * BB) const { 188f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineFunction * MF = BB->getParent(); 189f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineRegisterInfo &MRI = MF->getRegInfo(); 190f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineBasicBlock::iterator I = *MI; 191b5632b5b456db647b42239cbd4d8b58c82290c4eBill Wendling const R600InstrInfo *TII = 192b5632b5b456db647b42239cbd4d8b58c82290c4eBill Wendling static_cast<const R600InstrInfo*>(MF->getTarget().getInstrInfo()); 193f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 194f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (MI->getOpcode()) { 195a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard default: 19619a99df130f5747da950faf4ca5170d71f05594cTom Stellard // Replace LDS_*_RET instruction that don't have any uses with the 19719a99df130f5747da950faf4ca5170d71f05594cTom Stellard // equivalent LDS_*_NORET instruction. 19819a99df130f5747da950faf4ca5170d71f05594cTom Stellard if (TII->isLDSRetInstr(MI->getOpcode())) { 19979916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard int DstIdx = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst); 20079916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard assert(DstIdx != -1); 20179916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard MachineInstrBuilder NewMI; 20219a99df130f5747da950faf4ca5170d71f05594cTom Stellard if (!MRI.use_empty(MI->getOperand(DstIdx).getReg())) 20319a99df130f5747da950faf4ca5170d71f05594cTom Stellard return BB; 20419a99df130f5747da950faf4ca5170d71f05594cTom Stellard 20519a99df130f5747da950faf4ca5170d71f05594cTom Stellard NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), 20619a99df130f5747da950faf4ca5170d71f05594cTom Stellard TII->get(AMDGPU::getLDSNoRetOp(MI->getOpcode()))); 207a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard for (unsigned i = 1, e = MI->getNumOperands(); i < e; ++i) { 208a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard NewMI.addOperand(MI->getOperand(i)); 209a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } 210a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } else { 211a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 212a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } 213a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard break; 214f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::CLAMP_R600: { 215f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 216f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 217f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 218f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 219f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_CLAMP); 220f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 221f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 222f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 223f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::FABS_R600: { 224f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 225f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 226f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 227f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 228f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_ABS); 229f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 230f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 231f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 232f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::FNEG_R600: { 233f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 234f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 235f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 236f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 237f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_NEG); 238f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 239f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 240f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 241f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MASK_WRITE: { 242f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned maskedRegister = MI->getOperand(0).getReg(); 243f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(TargetRegisterInfo::isVirtualRegister(maskedRegister)); 244f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr * defInstr = MRI.getVRegDef(maskedRegister); 245f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(defInstr, 0, MO_FLAG_MASK); 246f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 247f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 248f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 249f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MOV_IMM_F32: 250f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 251f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getFPImm()->getValueAPF() 252f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .bitcastToAPInt().getZExtValue()); 253f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 254f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MOV_IMM_I32: 255f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 256f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getImm()); 257f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 258d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune case AMDGPU::CONST_COPY: { 259d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 260d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MI->getOperand(0).getReg(), AMDGPU::ALU_CONST); 2615e48a0e9ae2365a130dd1ec2e0b4beb337ab79e0Tom Stellard TII->setImmOperand(NewMI, AMDGPU::OpName::src0_sel, 262d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MI->getOperand(1).getImm()); 263d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune break; 264d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune } 265f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 266f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::RAT_WRITE_CACHELESS_32_eg: 267692ee102ebef535d311c35d53457028083e5c5beTom Stellard case AMDGPU::RAT_WRITE_CACHELESS_64_eg: 268f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::RAT_WRITE_CACHELESS_128_eg: { 26936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned EOP = (std::next(I)->getOpcode() == AMDGPU::RETURN) ? 1 : 0; 270f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 271f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 272f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 273f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 274f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(EOP); // Set End of program bit 275f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 276f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 277f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 278f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::TXD: { 279f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 280f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 281d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &RID = MI->getOperand(4); 282d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &SID = MI->getOperand(5); 283d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureId = MI->getOperand(6).getImm(); 284d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 285d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 286d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 287d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (TextureId) { 288d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 5: // Rect 289d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 290d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 291d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 6: // Shadow1D 292d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 293d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 294d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 7: // Shadow2D 295d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 296d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 297d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 8: // ShadowRect 298d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 299d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 300d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 301d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 9: // 1DArray 302d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 303d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 304d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 305d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 10: // 2DArray 306d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 307d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 308d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 11: // Shadow1DArray 309d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 310d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 311d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 312d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 12: // Shadow2DArray 313d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 314d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 315d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 316f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 317f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 318d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 319d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 320d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 321d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 322d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 323d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 324d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 325d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 326d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 327d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 328d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 329d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 330d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 331d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 332d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 333d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 334d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 335f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 336f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 337d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 338d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 339d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 340d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 341d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 342d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 343d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 344d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 345d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 346d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 347d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 348d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 349d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 350d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 351d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 352d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 353d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 354f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_G)) 355f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 356f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 357d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 358d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 359d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 360d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 361d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 362d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 363d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 364d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 365d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 366d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 367d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 368d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 369d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 370d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 371d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 372d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 373d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW) 374f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T0, RegState::Implicit) 375f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T1, RegState::Implicit); 376f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 377f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 378f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 379f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::TXD_SHADOW: { 380f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 381f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 382d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &RID = MI->getOperand(4); 383d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &SID = MI->getOperand(5); 384d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureId = MI->getOperand(6).getImm(); 385d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 386d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 387d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 388d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (TextureId) { 389d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 5: // Rect 390d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 391d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 392d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 6: // Shadow1D 393d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 394d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 395d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 7: // Shadow2D 396d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 397d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 398d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 8: // ShadowRect 399d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 400d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 401d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 402d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 9: // 1DArray 403d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 404d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 405d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 406d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 10: // 2DArray 407d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 408d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 409d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 11: // Shadow1DArray 410d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 411d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 412d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 413d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 12: // Shadow2DArray 414d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 415d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 416d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 417f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 418f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 419f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 420d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 421d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 422d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 423d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 424d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 425d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 426d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 427d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 428d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 429d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 430d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 431d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 432d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 433d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 434d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 435d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 436d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 437f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 438f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 439d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 440d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 441d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 442d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 443d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 444d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 445d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 446d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 447d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 448d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 449d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 450d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 451d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 452d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 453d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 454d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 455d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 456f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_C_G)) 457f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 458f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 459d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 460d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 461d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 462d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 463d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 464d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 465d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 466d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 467d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 468d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 469d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 470d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 471d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 472d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 473d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 474d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 475d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW) 476f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T0, RegState::Implicit) 477f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T1, RegState::Implicit); 478f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 479f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 480f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 481f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH: 482f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP)) 483fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune .addOperand(MI->getOperand(0)); 484f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 485f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 486f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH_COND_f32: { 487f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = 488f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 489f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::PREDICATE_BIT) 490f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 491f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(OPCODE_IS_NOT_ZERO) 492f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(0); // Flags 493f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 494fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 495f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 496f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 497f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 498f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 499f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 500f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH_COND_i32: { 501f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = 502f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 503f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::PREDICATE_BIT) 504f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 505f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(OPCODE_IS_NOT_ZERO_INT) 506f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(0); // Flags 507f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 508fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 509f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 510f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 511f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 512f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 513f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 514f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::EG_ExportSwz: 515f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::R600_ExportSwz: { 516254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard // Instruction is left unmodified if its not the last one of its type 517254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard bool isLastInstructionOfItsType = true; 518254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard unsigned InstExportType = MI->getOperand(1).getImm(); 51936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (MachineBasicBlock::iterator NextExportInst = std::next(I), 520254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard EndBlock = BB->end(); NextExportInst != EndBlock; 52136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NextExportInst = std::next(NextExportInst)) { 522254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (NextExportInst->getOpcode() == AMDGPU::EG_ExportSwz || 523254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard NextExportInst->getOpcode() == AMDGPU::R600_ExportSwz) { 524254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard unsigned CurrentInstExportType = NextExportInst->getOperand(1) 525254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard .getImm(); 526254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (CurrentInstExportType == InstExportType) { 527254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard isLastInstructionOfItsType = false; 528254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard break; 529254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 530254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 531254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 53236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EOP = (std::next(I)->getOpcode() == AMDGPU::RETURN) ? 1 : 0; 533254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (!EOP && !isLastInstructionOfItsType) 534f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return BB; 535f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned CfInst = (MI->getOpcode() == AMDGPU::EG_ExportSwz)? 84 : 40; 536f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 537f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 538f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 539f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 540f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 541f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(4)) 542f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(5)) 543f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(6)) 544f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(CfInst) 545254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard .addImm(EOP); 546f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 547f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 548a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen case AMDGPU::RETURN: { 549a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen // RETURN instructions must have the live-out registers as implicit uses, 550a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen // otherwise they appear dead. 551a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen R600MachineFunctionInfo *MFI = MF->getInfo<R600MachineFunctionInfo>(); 552a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MachineInstrBuilder MIB(*MF, MI); 553a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen for (unsigned i = 0, e = MFI->LiveOuts.size(); i != e; ++i) 554a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MIB.addReg(MFI->LiveOuts[i], RegState::Implicit); 555a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen return BB; 556a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen } 557f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 558f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 559f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->eraseFromParent(); 560f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return BB; 561f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 562f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 563f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 564f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Custom DAG Lowering Operations 565f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 566f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 567f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 568e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 569e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 570f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Op.getOpcode()) { 571f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 572cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 573cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 574cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG); 575cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SRA_PARTS: 576cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG); 577c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FCOS: 578c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FSIN: return LowerTrig(Op, DAG); 579f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 580f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::STORE: return LowerSTORE(Op, DAG); 581cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::LOAD: { 582cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Result = LowerLOAD(Op, DAG); 583cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines assert((!Result.getNode() || 584cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Result.getNode()->getNumValues() == 2) && 585cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines "Load should return a value and a chain"); 586cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return Result; 587cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 588cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 589cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::BRCOND: return LowerBRCOND(Op, DAG); 590e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG); 591f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_VOID: { 592f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain = Op.getOperand(0); 593f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = 594f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 595f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (IntrinsicID) { 596f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_store_output: { 597f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 598f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex); 599a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MFI->LiveOuts.push_back(Reg); 600ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getCopyToReg(Chain, SDLoc(Op), Reg, Op.getOperand(2)); 601f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 602abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune case AMDGPUIntrinsic::R600_store_swizzle: { 603abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune const SDValue Args[8] = { 604abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Chain, 605abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(2), // Export Value 606abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(3), // ArrayBase 607abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(4), // Type 608abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune DAG.getConstant(0, MVT::i32), // SWZ_X 609abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune DAG.getConstant(1, MVT::i32), // SWZ_Y 610abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune DAG.getConstant(2, MVT::i32), // SWZ_Z 611abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune DAG.getConstant(3, MVT::i32) // SWZ_W 612abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune }; 613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::EXPORT, SDLoc(Op), Op.getValueType(), Args); 614f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 615f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 616f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // default for switch(IntrinsicID) 617f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: break; 618f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 619f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode()) 620f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 621f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 622f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_WO_CHAIN: { 623f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = 624f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 625f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 626ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 627f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch(IntrinsicID) { 628f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 629837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune case AMDGPUIntrinsic::R600_load_input: { 630837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 631837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex); 632837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 633837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineRegisterInfo &MRI = MF.getRegInfo(); 634837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(Reg); 635837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return DAG.getCopyFromReg(DAG.getEntryNode(), 636837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), Reg, VT); 637837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 638837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune 639837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune case AMDGPUIntrinsic::R600_interp_input: { 640837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 641837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int ijb = cast<ConstantSDNode>(Op.getOperand(2))->getSExtValue(); 642837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineSDNode *interp; 643837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune if (ijb < 0) { 644837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune const MachineFunction &MF = DAG.getMachineFunction(); 645837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune const R600InstrInfo *TII = 646837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune static_cast<const R600InstrInfo*>(MF.getTarget().getInstrInfo()); 647837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_VEC_LOAD, DL, 648837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MVT::v4f32, DAG.getTargetConstant(slot / 4 , MVT::i32)); 649837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return DAG.getTargetExtractSubreg( 650837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune TII->getRegisterInfo().getSubRegFromChannel(slot % 4), 651837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune DL, MVT::f32, SDValue(interp, 0)); 652837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 653837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 654837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineRegisterInfo &MRI = MF.getRegInfo(); 655837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned RegisterI = AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb); 656837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned RegisterJ = AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb + 1); 657837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(RegisterI); 658837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(RegisterJ); 659837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDValue RegisterINode = DAG.getCopyFromReg(DAG.getEntryNode(), 660837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), RegisterI, MVT::f32); 661837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDValue RegisterJNode = DAG.getCopyFromReg(DAG.getEntryNode(), 662837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), RegisterJ, MVT::f32); 663837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune 664837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune if (slot % 4 < 2) 665837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 666837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32), 667837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune RegisterJNode, RegisterINode); 668837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune else 669837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 670837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32), 671837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune RegisterJNode, RegisterINode); 672837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return SDValue(interp, slot % 2); 673837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 67470a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune case AMDGPUIntrinsic::R600_interp_xy: 67570a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune case AMDGPUIntrinsic::R600_interp_zw: { 676f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 67729b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard MachineSDNode *interp; 67870a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue RegisterINode = Op.getOperand(2); 67970a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue RegisterJNode = Op.getOperand(3); 68029b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard 68170a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune if (IntrinsicID == AMDGPUIntrinsic::R600_interp_xy) 68229b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 68370a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune MVT::f32, MVT::f32, DAG.getTargetConstant(slot, MVT::i32), 6840962e147a439785279c3665379189017e980e0ccVincent Lejeune RegisterJNode, RegisterINode); 68529b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard else 68629b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 68770a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune MVT::f32, MVT::f32, DAG.getTargetConstant(slot, MVT::i32), 6880962e147a439785279c3665379189017e980e0ccVincent Lejeune RegisterJNode, RegisterINode); 68970a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, 69070a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue(interp, 0), SDValue(interp, 1)); 691f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 692d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_tex: 693d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_texc: 694d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txl: 695d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txlc: 696d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txb: 697d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txbc: 698d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txf: 699d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txq: 700d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddx: 701a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ddy: 702a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ldptr: { 703d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureOp; 704d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (IntrinsicID) { 705d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_tex: 706d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 0; 707d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 708d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_texc: 709d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 1; 710d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 711d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txl: 712d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 2; 713d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 714d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txlc: 715d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 3; 716d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 717d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txb: 718d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 4; 719d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 720d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txbc: 721d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 5; 722d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 723d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txf: 724d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 6; 725d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 726d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txq: 727d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 7; 728d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 729d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddx: 730d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 8; 731d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 732d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddy: 733d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 9; 734d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 735a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ldptr: 736a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune TextureOp = 10; 737a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune break; 738d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune default: 739d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune llvm_unreachable("Unknow Texture Operation"); 740d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 741d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 742d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SDValue TexArgs[19] = { 743d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(TextureOp, MVT::i32), 744d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(1), 745d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(0, MVT::i32), 746d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(1, MVT::i32), 747d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(2, MVT::i32), 748d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(3, MVT::i32), 749d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(2), 750d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(3), 751d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(4), 752d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(0, MVT::i32), 753d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(1, MVT::i32), 754d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(2, MVT::i32), 755d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune DAG.getConstant(3, MVT::i32), 756d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(5), 757d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(6), 758d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(7), 759d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(8), 760d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(9), 761d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(10) 762d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune }; 763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs); 764d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 7654ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune case AMDGPUIntrinsic::AMDGPU_dp4: { 7664ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune SDValue Args[8] = { 7674ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7684ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(0, MVT::i32)), 7694ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7704ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(0, MVT::i32)), 7714ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7724ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(1, MVT::i32)), 7734ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7744ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(1, MVT::i32)), 7754ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7764ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(2, MVT::i32)), 7774ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7784ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(2, MVT::i32)), 7794ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7804ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(3, MVT::i32)), 7814ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7824ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getConstant(3, MVT::i32)) 7834ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune }; 784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args); 7854ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune } 786f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 7877849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_x: 788f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 0); 7897849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_y: 790f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 1); 7917849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_z: 792f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 2); 7937849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_x: 794f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 3); 7957849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_y: 796f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 4); 7977849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_z: 798f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 5); 7997849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_x: 800f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 6); 8017849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_y: 802f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 7); 8037849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_z: 804f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 8); 805f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 8067849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_x: 807f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 808f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_X, VT); 8097849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_y: 810f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 811f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_Y, VT); 8127849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_z: 813f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 814f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_Z, VT); 8157849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_x: 816f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 817f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_X, VT); 8187849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_y: 819f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 820f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_Y, VT); 8217849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_z: 822f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 823f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_Z, VT); 824cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case Intrinsic::AMDGPU_rsq: 825cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // XXX - I'm assuming SI's RSQ_LEGACY matches R600's behavior. 826cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 827f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 828f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode()) 829f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 830f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 831f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } // end switch(Op.getOpcode()) 832f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return SDValue(); 833f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 834f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 835f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardvoid R600TargetLowering::ReplaceNodeResults(SDNode *N, 836f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVectorImpl<SDValue> &Results, 837f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 838f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (N->getOpcode()) { 83936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 84036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG); 84136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 842f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::FP_TO_UINT: Results.push_back(LowerFPTOUINT(N->getOperand(0), DAG)); 8439f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return; 844cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::UDIV: { 845cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 0); 846cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 847cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 848cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), 849cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N->getOperand(0), N->getOperand(1)); 850cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(UDIVREM); 851cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 8529f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 853cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::UREM: { 854cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 0); 855cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 856cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 857cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), 858cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N->getOperand(0), N->getOperand(1)); 859cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(UDIVREM.getValue(1)); 860cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 861cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 862cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SDIV: { 863cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 0); 864cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 865cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 866cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue SDIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(VT, VT), 867cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N->getOperand(0), N->getOperand(1)); 868cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(SDIVREM); 869cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 870cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 871cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SREM: { 872cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 0); 873cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 874cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 875cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue SDIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(VT, VT), 876cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines N->getOperand(0), N->getOperand(1)); 877cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(SDIVREM.getValue(1)); 878cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 879cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 880cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SDIVREM: { 881cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 1); 882cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RES = LowerSDIVREM(Op, DAG); 883cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(RES); 884cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(RES.getValue(1)); 885cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 886cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 887cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::UDIVREM: { 888cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Op = SDValue(N, 0); 889cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 890cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 891cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); 892cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 893cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue one = DAG.getConstant(1, HalfVT); 894cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue zero = DAG.getConstant(0, HalfVT); 895cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 896cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines //HiLo split 897cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS = N->getOperand(0); 898cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, zero); 899cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, one); 900cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 901cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS = N->getOperand(1); 902cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, zero); 903cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, one); 904cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 905cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Get Speculative values 906cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); 907cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); 908cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 909cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM_Hi = zero; 910cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, zero, REM_Part, LHS_Hi, ISD::SETEQ); 911cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 912cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, zero, DIV_Part, zero, ISD::SETEQ); 913cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DIV_Lo = zero; 914cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 915cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const unsigned halfBitWidth = HalfVT.getSizeInBits(); 916cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 917cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (unsigned i = 0; i < halfBitWidth; ++i) { 918cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue POS = DAG.getConstant(halfBitWidth - i - 1, HalfVT); 919cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Get Value of high bit 920cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue HBit; 921cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (halfBitWidth == 32 && Subtarget->hasBFE()) { 922cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines HBit = DAG.getNode(AMDGPUISD::BFE_U32, DL, HalfVT, LHS_Lo, POS, one); 923cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } else { 924cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); 925cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, one); 926cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 927cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 928cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Carry = DAG.getNode(ISD::SRL, DL, HalfVT, REM_Lo, 929cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getConstant(halfBitWidth - 1, HalfVT)); 930cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Hi = DAG.getNode(ISD::SHL, DL, HalfVT, REM_Hi, one); 931cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Hi = DAG.getNode(ISD::OR, DL, HalfVT, REM_Hi, Carry); 932cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 933cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Lo = DAG.getNode(ISD::SHL, DL, HalfVT, REM_Lo, one); 934cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Lo = DAG.getNode(ISD::OR, DL, HalfVT, REM_Lo, HBit); 935cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 936cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 937cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, REM_Lo, REM_Hi); 938cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 939cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue BIT = DAG.getConstant(1 << (halfBitWidth - i - 1), HalfVT); 940cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, zero, ISD::SETGE); 941cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 942cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); 943cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 944cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Update REM 945cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 946cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); 947cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 948cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETGE); 949cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, REM, zero); 950cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines REM_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, REM, one); 951cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 952cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 953cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, REM_Lo, REM_Hi); 954cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue DIV = DAG.getNode(ISD::BUILD_PAIR, DL, VT, DIV_Lo, DIV_Hi); 955cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(DIV); 956cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Results.push_back(REM); 957cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 958f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 959cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 960cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 961cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 962cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG, 963cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Vector) const { 964cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 965cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Vector); 966cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VecVT = Vector.getValueType(); 967cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT EltVT = VecVT.getVectorElementType(); 968cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SmallVector<SDValue, 8> Args; 969cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 970cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (unsigned i = 0, e = VecVT.getVectorNumElements(); 971cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines i != e; ++i) { 972cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 973cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector, DAG.getConstant(i, getVectorIdxTy()))); 974cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 975cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 976cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args); 977cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 978cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 979cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 980cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SelectionDAG &DAG) const { 981cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 982cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 983cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Vector = Op.getOperand(0); 984cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Index = Op.getOperand(1); 985cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 986cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (isa<ConstantSDNode>(Index) || 987cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 988cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return Op; 989cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 990cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector = vectorToVerticalVector(DAG, Vector); 991cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), 992cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector, Index); 993cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 994cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 995cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 996cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SelectionDAG &DAG) const { 997cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 998cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Vector = Op.getOperand(0); 999cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Value = Op.getOperand(1); 1000cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Index = Op.getOperand(2); 1001cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1002cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (isa<ConstantSDNode>(Index) || 1003cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 1004cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return Op; 1005cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1006cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector = vectorToVerticalVector(DAG, Vector); 1007cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(), 1008cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Vector, Value, Index); 1009cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return vectorToVerticalVector(DAG, Insert); 1010f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1011f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1012c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent LejeuneSDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 1013c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // On hw >= R700, COS/SIN input must be between -1. and 1. 1014c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) 1015c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune EVT VT = Op.getValueType(); 1016c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune SDValue Arg = Op.getOperand(0); 1017c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, SDLoc(Op), VT, 1018c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getNode(ISD::FADD, SDLoc(Op), VT, 1019c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getNode(ISD::FMUL, SDLoc(Op), VT, Arg, 1020c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getConstantFP(0.15915494309, MVT::f32)), 1021c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getConstantFP(0.5, MVT::f32))); 1022c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune unsigned TrigNode; 1023c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune switch (Op.getOpcode()) { 1024c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FCOS: 1025c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune TrigNode = AMDGPUISD::COS_HW; 1026c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune break; 1027c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FSIN: 1028c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune TrigNode = AMDGPUISD::SIN_HW; 1029c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune break; 1030c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune default: 1031c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune llvm_unreachable("Wrong trig opcode"); 1032c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune } 1033c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune SDValue TrigVal = DAG.getNode(TrigNode, SDLoc(Op), VT, 1034c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getNode(ISD::FADD, SDLoc(Op), VT, FractPart, 1035c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getConstantFP(-0.5, MVT::f32))); 1036c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune if (Gen >= AMDGPUSubtarget::R700) 1037c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune return TrigVal; 1038c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // On R600 hw, COS/SIN input must be between -Pi and Pi. 1039c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune return DAG.getNode(ISD::FMUL, SDLoc(Op), VT, TrigVal, 1040c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune DAG.getConstantFP(3.14159265359, MVT::f32)); 1041c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune} 1042c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune 1043cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const { 1044cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1045cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 1046cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1047cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Lo = Op.getOperand(0); 1048cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Hi = Op.getOperand(1); 1049cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Shift = Op.getOperand(2); 1050cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Zero = DAG.getConstant(0, VT); 1051cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue One = DAG.getConstant(1, VT); 1052cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1053cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Width = DAG.getConstant(VT.getSizeInBits(), VT); 1054cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, VT); 1055cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1056cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1057cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1058cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The dance around Width1 is necessary for 0 special case. 1059cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Without it the CompShift might be 32, producing incorrect results in 1060cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Overflow. So we do the shift in two steps, the alternative is to 1061cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // add a conditional to filter the special case. 1062cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1063cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift); 1064cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Overflow = DAG.getNode(ISD::SRL, DL, VT, Overflow, One); 1065cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1066cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift); 1067cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines HiSmall = DAG.getNode(ISD::OR, DL, VT, HiSmall, Overflow); 1068cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift); 1069cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1070cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift); 1071cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LoBig = Zero; 1072cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1073cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1074cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1075cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1076cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1077cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1078cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1079cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const { 1080cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 1081cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 1082cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1083cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Lo = Op.getOperand(0); 1084cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Hi = Op.getOperand(1); 1085cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Shift = Op.getOperand(2); 1086cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Zero = DAG.getConstant(0, VT); 1087cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue One = DAG.getConstant(1, VT); 1088cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1089cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines const bool SRA = Op.getOpcode() == ISD::SRA_PARTS; 1090cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1091cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Width = DAG.getConstant(VT.getSizeInBits(), VT); 1092cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, VT); 1093cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1094cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1095cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1096cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The dance around Width1 is necessary for 0 special case. 1097cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Without it the CompShift might be 32, producing incorrect results in 1098cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Overflow. So we do the shift in two steps, the alternative is to 1099cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // add a conditional to filter the special case. 1100cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1101cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift); 1102cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Overflow = DAG.getNode(ISD::SHL, DL, VT, Overflow, One); 1103cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1104cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift); 1105cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift); 1106cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines LoSmall = DAG.getNode(ISD::OR, DL, VT, LoSmall, Overflow); 1107cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1108cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift); 1109cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero; 1110cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1111cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1112cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1113cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1114cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1115cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1116cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1117f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const { 1118f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode( 1119f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETCC, 1120ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc(Op), 1121f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MVT::i1, 1122f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op, DAG.getConstantFP(0.0f, MVT::f32), 1123f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(ISD::SETNE) 1124f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ); 1125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1126f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1127f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT, 1128ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, 1129f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned DwordOffset) const { 1130f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned ByteOffset = DwordOffset * 4; 1131f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1132a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard AMDGPUAS::CONSTANT_BUFFER_0); 1133f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1134f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // We shouldn't be using an offset wider than 16-bits for implicit parameters. 1135f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(isInt<16>(ByteOffset)); 1136f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1137f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getLoad(VT, DL, DAG.getEntryNode(), 1138f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(ByteOffset, MVT::i32), // PTR 1139f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachinePointerInfo(ConstantPointerNull::get(PtrType)), 1140f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard false, false, false, 0); 1141f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1142f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1143f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool R600TargetLowering::isZero(SDValue Op) const { 1144f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) { 1145f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Cst->isNullValue(); 1146f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){ 1147f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CstFP->isZero(); 1148f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else { 1149f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 1150f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1151f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1152f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1153f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 1154ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 1155f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 1156f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1157f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue LHS = Op.getOperand(0); 1158f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RHS = Op.getOperand(1); 1159f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue True = Op.getOperand(2); 1160f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue False = Op.getOperand(3); 1161f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue CC = Op.getOperand(4); 1162f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Temp; 1163f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1164f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // LHS and RHS are guaranteed to be the same value type 1165f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT CompareVT = LHS.getValueType(); 1166f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1167f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Check if we can lower this to a native operation. 1168f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 11691454cb86be54a8703fca396752be71c50c805b88Tom Stellard // Try to lower to a SET* instruction: 11701454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 11711454cb86be54a8703fca396752be71c50c805b88Tom Stellard // SET* can match the following patterns: 11721454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 117312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, f32, -1, 0, cc_supported 117412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, f32, 1.0f, 0.0f, cc_supported 117512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, i32, -1, 0, cc_supported 11761454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 11771454cb86be54a8703fca396752be71c50c805b88Tom Stellard 11781454cb86be54a8703fca396752be71c50c805b88Tom Stellard // Move hardware True/False values to the correct operand. 117912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 118012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode InverseCC = 118112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1182bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isHWTrueValue(False) && isHWFalseValue(True)) { 1183bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { 1184bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(False, True); 1185bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard CC = DAG.getCondCode(InverseCC); 1186bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } else { 1187bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC); 1188bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) { 1189bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(False, True); 1190bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(LHS, RHS); 1191bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard CC = DAG.getCondCode(SwapInvCC); 1192bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } 1193bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } 11941454cb86be54a8703fca396752be71c50c805b88Tom Stellard } 11951454cb86be54a8703fca396752be71c50c805b88Tom Stellard 11961454cb86be54a8703fca396752be71c50c805b88Tom Stellard if (isHWTrueValue(True) && isHWFalseValue(False) && 11971454cb86be54a8703fca396752be71c50c805b88Tom Stellard (CompareVT == VT || VT == MVT::i32)) { 11981454cb86be54a8703fca396752be71c50c805b88Tom Stellard // This can be matched by a SET* instruction. 11991454cb86be54a8703fca396752be71c50c805b88Tom Stellard return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC); 12001454cb86be54a8703fca396752be71c50c805b88Tom Stellard } 12011454cb86be54a8703fca396752be71c50c805b88Tom Stellard 1202f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Try to lower to a CND* instruction: 12031454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 12041454cb86be54a8703fca396752be71c50c805b88Tom Stellard // CND* can match the following patterns: 12051454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 120612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, 0.0, f32, f32, cc_supported 120712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, 0.0, i32, i32, cc_supported 120812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, 0, f32, f32, cc_supported 120912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, 0, i32, i32, cc_supported 12101454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 121112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 121212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try to move the zero value to the RHS 121312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isZero(LHS)) { 121412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 121512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try swapping the operands 121612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode); 121712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 121812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(LHS, RHS); 121912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CC = DAG.getCondCode(CCSwapped); 122012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } else { 122112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try inverting the conditon and then swapping the operands 122212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT.isInteger()); 122312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CCSwapped = ISD::getSetCCSwappedOperands(CCInv); 122412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 122512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(True, False); 122612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(LHS, RHS); 122712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CC = DAG.getCondCode(CCSwapped); 122812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 122912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 123012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 123112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isZero(RHS)) { 123212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard SDValue Cond = LHS; 123312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard SDValue Zero = RHS; 1234f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1235f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (CompareVT != VT) { 1236f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Bitcast True / False to the correct types. This will end up being 1237f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // a nop, but it allows us to define only a single pattern in the 1238f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // .TD files for each CND* instruction rather than having to have 1239f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // one pattern for integer True/False and one for fp True/False 1240f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True); 1241f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False); 1242f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1243f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1244f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (CCOpcode) { 1245f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETONE: 1246f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUNE: 1247f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETNE: 1248f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1249f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Temp = True; 1250f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True = False; 1251f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard False = Temp; 1252f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1253f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: 1254f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1255f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1256f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, 1257f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Cond, Zero, 1258f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True, False, 1259f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(CCOpcode)); 1260f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode); 1261f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1262f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1263f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // If we make it this for it means we have no native instructions to handle 1264f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // this SELECT_CC, so we must lower it. 1265f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue HWTrue, HWFalse; 1266f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1267f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (CompareVT == MVT::f32) { 1268f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard HWTrue = DAG.getConstantFP(1.0f, CompareVT); 1269f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard HWFalse = DAG.getConstantFP(0.0f, CompareVT); 1270f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else if (CompareVT == MVT::i32) { 1271f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard HWTrue = DAG.getConstant(-1, CompareVT); 1272f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard HWFalse = DAG.getConstant(0, CompareVT); 1273f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1274f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else { 127536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Unhandled value type in LowerSELECT_CC"); 1276f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1277f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1278f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Lower this unsupported SELECT_CC into a combination of two supported 1279f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // SELECT_CC operations. 1280f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC); 1281f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1282f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::SELECT_CC, DL, VT, 1283f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Cond, HWFalse, 1284f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True, False, 1285f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(ISD::SETNE)); 1286f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1287f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 128836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// LLVM generates byte-addressed pointers. For indirect addressing, we need to 1289c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// convert these pointers to a register index. Each register holds 1290c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// 16 bytes, (4 x 32bit sub-register), but we need to take into account the 1291c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// \p StackWidth, which tells us how many of the 4 sub-registrers will be used 1292c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// for indirect addressing. 1293c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom StellardSDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr, 1294c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth, 1295c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SelectionDAG &DAG) const { 1296c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned SRLPad; 1297c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard switch(StackWidth) { 1298c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 1: 1299c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 2; 1300c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1301c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 2: 1302c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 3; 1303c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1304c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 4: 1305c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 4; 1306c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1307c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard default: llvm_unreachable("Invalid stack width"); 1308c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1309c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1310ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SRL, SDLoc(Ptr), Ptr.getValueType(), Ptr, 1311c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getConstant(SRLPad, MVT::i32)); 1312c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard} 1313c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1314c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellardvoid R600TargetLowering::getStackAddress(unsigned StackWidth, 1315c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned ElemIdx, 1316c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned &Channel, 1317c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned &PtrIncr) const { 1318c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard switch (StackWidth) { 1319c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard default: 1320c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 1: 1321c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = 0; 1322c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ElemIdx > 0) { 1323c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 1; 1324c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1325c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1326c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1327c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1328c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 2: 1329c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = ElemIdx % 2; 1330c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ElemIdx == 2) { 1331c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 1; 1332c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1333c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1334c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1335c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1336c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 4: 1337c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = ElemIdx; 1338c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1339c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1340c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1341c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard} 1342c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1343f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1344ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 1345f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard StoreSDNode *StoreNode = cast<StoreSDNode>(Op); 1346f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain = Op.getOperand(0); 1347f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Value = Op.getOperand(1); 1348f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Ptr = Op.getOperand(2); 1349f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 13507a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Result = AMDGPUTargetLowering::LowerSTORE(Op, DAG); 13514c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard if (Result.getNode()) { 13524c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard return Result; 13534c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard } 13544c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 1355ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS) { 1356ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->isTruncatingStore()) { 1357ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard EVT VT = Value.getValueType(); 13584c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard assert(VT.bitsLE(MVT::i32)); 1359ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard EVT MemVT = StoreNode->getMemoryVT(); 1360ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue MaskConstant; 1361ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (MemVT == MVT::i8) { 1362ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard MaskConstant = DAG.getConstant(0xFF, MVT::i32); 1363ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else { 1364ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard assert(MemVT == MVT::i16); 1365ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard MaskConstant = DAG.getConstant(0xFFFF, MVT::i32); 1366ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } 1367ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr, 1368ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getConstant(2, MVT::i32)); 1369ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr, 1370ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getConstant(0x00000003, VT)); 1371ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant); 1372ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex, 1373ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getConstant(3, VT)); 1374ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, Shift); 1375ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, Shift); 1376ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 1377ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // vector instead. 1378ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Src[4] = { 1379ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard ShiftedValue, 1380ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getConstant(0, MVT::i32), 1381ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getConstant(0, MVT::i32), 1382ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Mask 1383ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard }; 1384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Input = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i32, Src); 1385ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Args[3] = { Chain, Input, DWordAddr }; 1386ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL, 1387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op->getVTList(), Args, MemVT, 1388ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard StoreNode->getMemOperand()); 1389ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && 1390ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Value.getValueType().bitsGE(MVT::i32)) { 1391ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // Convert pointer from byte address to dword address. 1392ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(), 1393ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), 1394ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Ptr, DAG.getConstant(2, MVT::i32))); 1395ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard 1396ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) { 139736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Truncated and indexed stores not supported yet"); 1398ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else { 1399ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1400ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } 1401ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard return Chain; 1402f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1403f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1404c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1405c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ValueVT = Value.getValueType(); 1406c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1407c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (StoreNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1408c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return SDValue(); 1409c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1410c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 141136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ret = AMDGPUTargetLowering::LowerSTORE(Op, DAG); 141236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Ret.getNode()) { 141336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Ret; 141436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 1415c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Lowering for indirect addressing 1416c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1417c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const MachineFunction &MF = DAG.getMachineFunction(); 1418c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const AMDGPUFrameLowering *TFL = static_cast<const AMDGPUFrameLowering*>( 1419c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getTargetMachine().getFrameLowering()); 1420c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth = TFL->getStackWidth(MF); 1421c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1422c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1423c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1424c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ValueVT.isVector()) { 1425c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned NumElemVT = ValueVT.getVectorNumElements(); 1426c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ElemVT = ValueVT.getVectorElementType(); 1427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 4> Stores(NumElemVT); 1428c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1429c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1430c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard "vector width in load"); 1431c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1432c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = 0; i < NumElemVT; ++i) { 1433c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned Channel, PtrIncr; 1434c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getStackAddress(StackWidth, i, Channel, PtrIncr); 1435c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1436c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getConstant(PtrIncr, MVT::i32)); 1437c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SDValue Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, 1438c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Value, DAG.getConstant(i, MVT::i32)); 1439c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1440c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Stores[i] = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1441c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Elem, Ptr, 1442c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getTargetConstant(Channel, MVT::i32)); 1443c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); 1445c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1446c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ValueVT == MVT::i8) { 1447c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Value = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Value); 1448c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1449c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, Chain, Value, Ptr, 145051c2e124e3a559d5728acb4b6cd130e2129b4135NAKAMURA Takumi DAG.getTargetConstant(0, MVT::i32)); // Channel 1451c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1452c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1453c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return Chain; 1454f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1455f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 14569f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard// return (512 + (kc_bank << 12) 14579f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellardstatic int 14589f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom StellardConstantAddressBlock(unsigned AddressSpace) { 14599f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard switch (AddressSpace) { 14609f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_0: 14619f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512; 14629f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_1: 14639f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096; 14649f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_2: 14659f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 2; 14669f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_3: 14679f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 3; 14689f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_4: 14699f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 4; 14709f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_5: 14719f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 5; 14729f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_6: 14739f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 6; 14749f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_7: 14759f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 7; 14769f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_8: 14779f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 8; 14789f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_9: 14799f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 9; 14809f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_10: 14819f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 10; 14829f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_11: 14839f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 11; 14849f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_12: 14859f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 12; 14869f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_13: 14879f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 13; 14889f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_14: 14899f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 14; 14909f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_15: 14919f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 15; 14929f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard default: 14939f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return -1; 14949f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 14959f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard} 14969f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 14979f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom StellardSDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const 14989f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard{ 14999f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard EVT VT = Op.getValueType(); 1500ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 15019f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard LoadSDNode *LoadNode = cast<LoadSDNode>(Op); 15029f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Chain = Op.getOperand(0); 15039f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Ptr = Op.getOperand(1); 15049f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue LoweredLoad; 15059f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 150636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ret = AMDGPUTargetLowering::LowerLOAD(Op, DAG); 150736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Ret.getNode()) { 1508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { 1509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ret, 1510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain 1511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 1512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 151336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 151436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 151536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1516d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) { 1517d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDValue MergedValues[2] = { 1518d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SplitVectorLoad(Op, DAG), 1519d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Chain 1520d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard }; 1521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 1522d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard } 1523d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 15249f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); 152529f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault if (ConstantBlock > -1 && 152629f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || 152729f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { 15289f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Result; 1529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<ConstantExpr>(LoadNode->getMemOperand()->getValue()) || 1530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<Constant>(LoadNode->getMemOperand()->getValue()) || 15317a59defe70257302e2f68daba2fb8e1735e7cd0eMatt Arsenault isa<ConstantSDNode>(Ptr)) { 15329f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Slots[4]; 15339f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard for (unsigned i = 0; i < 4; i++) { 15349f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // We want Const position encoded with the following formula : 15359f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // (((512 + (kc_bank << 12) + const_index) << 2) + chan) 15369f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // const_index is Ptr computed by llvm using an alignment of 16. 15379f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and 15389f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // then div by 4 at the ISel step 15399f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 15409f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard DAG.getConstant(4 * i + ConstantBlock * 16, MVT::i32)); 15419f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr); 15429f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 1543692ee102ebef535d311c35d53457028083e5c5beTom Stellard EVT NewVT = MVT::v4i32; 1544692ee102ebef535d311c35d53457028083e5c5beTom Stellard unsigned NumElements = 4; 1545692ee102ebef535d311c35d53457028083e5c5beTom Stellard if (VT.isVector()) { 1546692ee102ebef535d311c35d53457028083e5c5beTom Stellard NewVT = VT; 1547692ee102ebef535d311c35d53457028083e5c5beTom Stellard NumElements = VT.getVectorNumElements(); 1548692ee102ebef535d311c35d53457028083e5c5beTom Stellard } 1549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result = DAG.getNode(ISD::BUILD_VECTOR, DL, NewVT, 1550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines makeArrayRef(Slots, NumElements)); 15519f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } else { 155236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // non-constant ptr can't be folded, keeps it as a v4f32 load 15539f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32, 15543f7f8e814ef49b79b9c41e75df40be3bdb3612f5Vincent Lejeune DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(4, MVT::i32)), 1555ff408c07282309a8a3d4daca7c7e127d2fce01edChristian Konig DAG.getConstant(LoadNode->getAddressSpace() - 155651c2e124e3a559d5728acb4b6cd130e2129b4135NAKAMURA Takumi AMDGPUAS::CONSTANT_BUFFER_0, MVT::i32) 15579f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard ); 15589f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15599f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 15609f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (!VT.isVector()) { 15619f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 15629f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard DAG.getConstant(0, MVT::i32)); 15639f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15649f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 15659f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue MergedValues[2] = { 1566dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result, 1567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain 15689f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard }; 1569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 15709f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15719f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 1572dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // For most operations returning SDValue() will result in the node being 1573dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we 1574dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // need to manually expand loads that may be legal in some address spaces and 1575dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for 1576dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // compute shaders, since the data is sign extended when it is uploaded to the 1577dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // buffer. However SEXT loads from other address spaces are not supported, so 1578dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // we need to expand them here. 157958d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { 158058d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard EVT MemVT = LoadNode->getMemoryVT(); 158158d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); 158258d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue ShiftAmount = 158358d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard DAG.getConstant(VT.getSizeInBits() - MemVT.getSizeInBits(), MVT::i32); 158458d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue NewLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, Chain, Ptr, 158558d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->getPointerInfo(), MemVT, 158658d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->isVolatile(), 158758d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->isNonTemporal(), 158858d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->getAlignment()); 158958d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, NewLoad, ShiftAmount); 159058d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Shl, ShiftAmount); 159158d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard 159258d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue MergedValues[2] = { Sra, Chain }; 1593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 159458d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard } 159558d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard 1596c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1597c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return SDValue(); 1598c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1599c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1600c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Lowering for indirect addressing 1601c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const MachineFunction &MF = DAG.getMachineFunction(); 1602c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const AMDGPUFrameLowering *TFL = static_cast<const AMDGPUFrameLowering*>( 1603c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getTargetMachine().getFrameLowering()); 1604c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth = TFL->getStackWidth(MF); 1605c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1606c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1607c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1608c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (VT.isVector()) { 1609c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned NumElemVT = VT.getVectorNumElements(); 1610c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ElemVT = VT.getVectorElementType(); 1611c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SDValue Loads[4]; 1612c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1613c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1614c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard "vector width in load"); 1615c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1616c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = 0; i < NumElemVT; ++i) { 1617c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned Channel, PtrIncr; 1618c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getStackAddress(StackWidth, i, Channel, PtrIncr); 1619c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 1620c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getConstant(PtrIncr, MVT::i32)); 1621c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Loads[i] = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, ElemVT, 1622c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Ptr, 1623c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getTargetConstant(Channel, MVT::i32), 1624c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Op.getOperand(2)); 1625c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1626c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = NumElemVT; i < 4; ++i) { 1627c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Loads[i] = DAG.getUNDEF(ElemVT); 1628c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1629c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT TargetVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 4); 1630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoweredLoad = DAG.getNode(ISD::BUILD_VECTOR, DL, TargetVT, Loads); 1631c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1632c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard LoweredLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, VT, 1633c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Ptr, 1634c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard DAG.getTargetConstant(0, MVT::i32), // Channel 1635c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Op.getOperand(2)); 1636c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1637c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { 1639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoweredLoad, 1640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain 1641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 1642c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 16449f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard} 1645f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1646cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 1647cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Chain = Op.getOperand(0); 1648cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Cond = Op.getOperand(1); 1649cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Jump = Op.getOperand(2); 1650cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1651cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(), 1652cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Chain, Jump, Cond); 1653cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 1654cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 1655f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// XXX Only kernel functions are supported, so we can assume for now that 1656f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// every function is a kernel function, but in the future we should use 1657f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// separate calling conventions for kernel and non-kernel functions. 1658f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerFormalArguments( 1659f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain, 1660f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CallingConv::ID CallConv, 1661f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard bool isVarArg, 1662f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<ISD::InputArg> &Ins, 1663ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, SelectionDAG &DAG, 1664f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVectorImpl<SDValue> &InVals) const { 1665f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard SmallVector<CCValAssign, 16> ArgLocs; 1666f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), 1667f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard getTargetMachine(), ArgLocs, *DAG.getContext()); 166870a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 166970a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune unsigned ShaderType = MF.getInfo<R600MachineFunctionInfo>()->ShaderType; 1670f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 1671f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard SmallVector<ISD::InputArg, 8> LocalIns; 1672f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 1673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines getOriginalFunctionArgs(DAG, MF.getFunction(), Ins, LocalIns); 1674f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 1675f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AnalyzeFormalArguments(CCInfo, LocalIns); 1676f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 1677a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 1678f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard CCValAssign &VA = ArgLocs[i]; 1679f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard EVT VT = Ins[i].VT; 1680f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard EVT MemVT = LocalIns[i].VT; 16815864284d71ed89a4280e5171c389ad83fe183db7Tom Stellard 168270a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune if (ShaderType != ShaderType::COMPUTE) { 168370a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune unsigned Reg = MF.addLiveIn(VA.getLocReg(), &AMDGPU::R600_Reg128RegClass); 168470a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT); 168570a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune InVals.push_back(Register); 168670a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune continue; 168770a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune } 168870a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune 1689f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1690a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard AMDGPUAS::CONSTANT_BUFFER_0); 1691f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 169236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // i64 isn't a legal type, so the register type used ends up as i32, which 169336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't expected here. It attempts to create this sextload, but it ends up 169436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // being invalid. Somehow this seems to work with i64 arguments, but breaks 169536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for <1 x i64>. 169636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1697f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard // The first 36 bytes of the input buffer contains information about 1698f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard // thread group and global sizes. 1699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This should really check the extload type, but the handling of 1701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // extload vecto parameters seems to be broken. 1702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines //ISD::LoadExtType Ext = Ins[i].Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD; 1703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::LoadExtType Ext = ISD::SEXTLOAD; 1704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Arg = DAG.getExtLoad(Ext, DL, VT, Chain, 1705f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard DAG.getConstant(36 + VA.getLocMemOffset(), MVT::i32), 1706f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard MachinePointerInfo(UndefValue::get(PtrTy)), 1707f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard MemVT, false, false, 4); 1708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 4 is the preferred alignment for the CONSTANT memory space. 1710f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard InVals.push_back(Arg); 1711f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1712f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Chain; 1713f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1714f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1715225ed7069caae9ece32d8bd3d15c6e41e21cc04bMatt ArsenaultEVT R600TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { 1716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 1717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MVT::i32; 1718f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return VT.changeVectorElementTypeToInteger(); 1719f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1720f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue CompactSwizzlableVector( 1722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG, SDValue VectorEntry, 1723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<unsigned, unsigned> &RemapSwizzle) { 172498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 172598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(RemapSwizzle.empty()); 172698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewBldVec[4] = { 1727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(0), 1728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(1), 1729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(2), 1730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(3) 173198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 173298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 173398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 1734f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::UNDEF) 1735f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // We mask write here to teach later passes that the ith element of this 1736f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // vector is undef. Thus we can use it to reduce 128 bits reg usage, 1737f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // break false dependencies and additionnaly make assembly easier to read. 1738f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune RemapSwizzle[i] = 7; // SEL_MASK_WRITE 173998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) { 174098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (C->isZero()) { 174198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = 4; // SEL_0 174298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(MVT::f32); 174398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } else if (C->isExactlyValue(1.0)) { 174498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = 5; // SEL_1 174598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(MVT::f32); 174698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 174798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 174898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 174998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::UNDEF) 175098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune continue; 175198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned j = 0; j < i; j++) { 175298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i] == NewBldVec[j]) { 175398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); 175498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = j; 175598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune break; 175698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 175798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 175898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 175998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 176098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(VectorEntry), 1761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getValueType(), NewBldVec); 176298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 176398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 17646c59c7a6fdc5b58de4f3c349b59a567c239818e4Benjamin Kramerstatic SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, 17656c59c7a6fdc5b58de4f3c349b59a567c239818e4Benjamin Kramer DenseMap<unsigned, unsigned> &RemapSwizzle) { 176698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 176798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(RemapSwizzle.empty()); 176898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewBldVec[4] = { 176998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(0), 177098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(1), 177198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(2), 177298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(3) 177398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 177498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune bool isUnmovable[4] = { false, false, false, false }; 177536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0; i < 4; i++) { 1776f4bdec2ebeb1306a77e9377583c5799199775f88Vincent Lejeune RemapSwizzle[i] = i; 177736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 177836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 177936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ->getZExtValue(); 178036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (i == Idx) 178136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isUnmovable[Idx] = true; 178236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 178598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 178698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 178798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 178898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune ->getZExtValue(); 178991ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune if (isUnmovable[Idx]) 179091ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune continue; 179191ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune // Swap i and Idx 179291ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune std::swap(NewBldVec[Idx], NewBldVec[i]); 179391ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune std::swap(RemapSwizzle[i], RemapSwizzle[Idx]); 179491ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune break; 179598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 179698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 179798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 179898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(VectorEntry), 1799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getValueType(), NewBldVec); 180098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 180198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 180298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 180398017a015bb862afce4f90d432eded4e28fe1d26Vincent LejeuneSDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, 180498017a015bb862afce4f90d432eded4e28fe1d26Vincent LejeuneSDValue Swz[4], SelectionDAG &DAG) const { 180598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(BuildVector.getOpcode() == ISD::BUILD_VECTOR); 180698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune // Old -> New swizzle values 180798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune DenseMap<unsigned, unsigned> SwizzleRemap; 180898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 180998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap); 181098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 181198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune unsigned Idx = dyn_cast<ConstantSDNode>(Swz[i])->getZExtValue(); 181298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 181398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune Swz[i] = DAG.getConstant(SwizzleRemap[Idx], MVT::i32); 181498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 181598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 181698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SwizzleRemap.clear(); 181798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap); 181898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 181998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune unsigned Idx = dyn_cast<ConstantSDNode>(Swz[i])->getZExtValue(); 182098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 182198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune Swz[i] = DAG.getConstant(SwizzleRemap[Idx], MVT::i32); 182298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 182398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 182498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return BuildVector; 182598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 182698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 182798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 1828f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 1829f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Custom DAG Optimizations 1830f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 1831f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1832f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::PerformDAGCombine(SDNode *N, 1833f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAGCombinerInfo &DCI) const { 1834f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG = DCI.DAG; 1835f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1836f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (N->getOpcode()) { 1837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1838f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) 1839f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::FP_ROUND: { 1840f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Arg = N->getOperand(0); 1841f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) { 1842ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), N->getValueType(0), 1843f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Arg.getOperand(0)); 1844f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1845f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1846f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 18471234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 18481234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> 18491234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // (i32 select_cc f32, f32, -1, 0 cc) 18501234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // 18511234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // Mesa's GLSL frontend generates the above pattern a lot and we can lower 18521234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // this to one of the SET*_DX10 instructions. 18531234c9be42b4ebd4b398df461123205dccf3706cTom Stellard case ISD::FP_TO_SINT: { 18541234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue FNeg = N->getOperand(0); 18551234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (FNeg.getOpcode() != ISD::FNEG) { 18561234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 18571234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 18581234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue SelectCC = FNeg.getOperand(0); 18591234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (SelectCC.getOpcode() != ISD::SELECT_CC || 18601234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS 18611234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(2).getValueType() != MVT::f32 || // True 18621234c9be42b4ebd4b398df461123205dccf3706cTom Stellard !isHWTrueValue(SelectCC.getOperand(2)) || 18631234c9be42b4ebd4b398df461123205dccf3706cTom Stellard !isHWFalseValue(SelectCC.getOperand(3))) { 18641234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 18651234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 18661234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 1867ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 18681234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(0), // LHS 18691234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(1), // RHS 18701234c9be42b4ebd4b398df461123205dccf3706cTom Stellard DAG.getConstant(-1, MVT::i32), // True 18711234c9be42b4ebd4b398df461123205dccf3706cTom Stellard DAG.getConstant(0, MVT::i32), // Flase 18721234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(4)); // CC 18731234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 18741234c9be42b4ebd4b398df461123205dccf3706cTom Stellard break; 18751234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 187615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 18771fe9069d53f586963d61523f7c5a7d41d80a9d8bNAKAMURA Takumi // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx 18781fe9069d53f586963d61523f7c5a7d41d80a9d8bNAKAMURA Takumi // => build_vector elt0, ... , NewEltIdx, ... , eltN 187915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet case ISD::INSERT_VECTOR_ELT: { 188015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue InVec = N->getOperand(0); 188115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue InVal = N->getOperand(1); 188215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue EltNo = N->getOperand(2); 188315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDLoc dl(N); 188415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 188515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // If the inserted element is an UNDEF, just use the input vector. 188615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVal.getOpcode() == ISD::UNDEF) 188715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return InVec; 188815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 188915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet EVT VT = InVec.getValueType(); 189015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 189115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // If we can't generate a legal BUILD_VECTOR, exit 189215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (!isOperationLegal(ISD::BUILD_VECTOR, VT)) 189315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 189415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 189515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Check that we know which element is being inserted 189615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (!isa<ConstantSDNode>(EltNo)) 189715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 189815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 189915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 190015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 190115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 190215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // vector elements. 190315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SmallVector<SDValue, 8> Ops; 190415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 190515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops.append(InVec.getNode()->op_begin(), 190615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet InVec.getNode()->op_end()); 190715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } else if (InVec.getOpcode() == ISD::UNDEF) { 190815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet unsigned NElts = VT.getVectorNumElements(); 190915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 191015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } else { 191115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 191215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 191315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 191415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Insert the element 191515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (Elt < Ops.size()) { 191615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // All the operands of BUILD_VECTOR must have the same type; 191715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // we enforce that here. 191815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet EVT OpVT = Ops[0].getValueType(); 191915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVal.getValueType() != OpVT) 192015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet InVal = OpVT.bitsGT(InVal.getValueType()) ? 192115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : 192215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); 192315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops[Elt] = InVal; 192415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 192515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 192615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Return the new vector 1927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 192815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 192915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 19309f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // Extract_vec (Build_vector) generated by custom lowering 19319f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // also needs to be customly combined 19329f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case ISD::EXTRACT_VECTOR_ELT: { 19339f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Arg = N->getOperand(0); 19349f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (Arg.getOpcode() == ISD::BUILD_VECTOR) { 19359f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 19369f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard unsigned Element = Const->getZExtValue(); 19379f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return Arg->getOperand(Element); 19389f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 19399f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 1940cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard if (Arg.getOpcode() == ISD::BITCAST && 1941cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { 1942cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 1943cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard unsigned Element = Const->getZExtValue(); 1944ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getVTList(), 1945cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard Arg->getOperand(0).getOperand(Element)); 1946cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard } 1947cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard } 19489f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 19491234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19501234c9be42b4ebd4b398df461123205dccf3706cTom Stellard case ISD::SELECT_CC: { 1951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Try common optimizations 1952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Ret.getNode()) 1954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Ret; 1955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 19561234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> 19571234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // selectcc x, y, a, b, inv(cc) 19587893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // 19597893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> 19607893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // selectcc x, y, a, b, cc 19611234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue LHS = N->getOperand(0); 19621234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (LHS.getOpcode() != ISD::SELECT_CC) { 19631234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 19641234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 19651234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19661234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue RHS = N->getOperand(1); 19671234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue True = N->getOperand(2); 19681234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue False = N->getOperand(3); 19697893d29c62146baddf43c4d9d42678d246a52feaTom Stellard ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 19701234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19711234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (LHS.getOperand(2).getNode() != True.getNode() || 19721234c9be42b4ebd4b398df461123205dccf3706cTom Stellard LHS.getOperand(3).getNode() != False.getNode() || 19737893d29c62146baddf43c4d9d42678d246a52feaTom Stellard RHS.getNode() != False.getNode()) { 19741234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 19751234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 19761234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19777893d29c62146baddf43c4d9d42678d246a52feaTom Stellard switch (NCC) { 19787893d29c62146baddf43c4d9d42678d246a52feaTom Stellard default: return SDValue(); 19797893d29c62146baddf43c4d9d42678d246a52feaTom Stellard case ISD::SETNE: return LHS; 19807893d29c62146baddf43c4d9d42678d246a52feaTom Stellard case ISD::SETEQ: { 19817893d29c62146baddf43c4d9d42678d246a52feaTom Stellard ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); 19827893d29c62146baddf43c4d9d42678d246a52feaTom Stellard LHSCC = ISD::getSetCCInverse(LHSCC, 19837893d29c62146baddf43c4d9d42678d246a52feaTom Stellard LHS.getOperand(0).getValueType().isInteger()); 198412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (DCI.isBeforeLegalizeOps() || 198512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) 198612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard return DAG.getSelectCC(SDLoc(N), 198712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(0), 198812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(1), 198912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(2), 199012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(3), 199112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHSCC); 199212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard break; 1993abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune } 19947893d29c62146baddf43c4d9d42678d246a52feaTom Stellard } 199512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard return SDValue(); 19967893d29c62146baddf43c4d9d42678d246a52feaTom Stellard } 19974c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 1998abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune case AMDGPUISD::EXPORT: { 1999abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue Arg = N->getOperand(1); 2000abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2001abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune break; 200298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 2003abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue NewArgs[8] = { 2004abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(0), // Chain 2005abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue(), 2006abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(2), // ArrayBase 2007abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(3), // Type 2008abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(4), // SWZ_X 2009abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(5), // SWZ_Y 2010abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(6), // SWZ_Z 2011abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(7) // SWZ_W 2012abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune }; 2013ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(N); 201498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG); 2015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::EXPORT, DL, N->getVTList(), NewArgs); 20161234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 201798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune case AMDGPUISD::TEXTURE_FETCH: { 201898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue Arg = N->getOperand(1); 201998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (Arg.getOpcode() != ISD::BUILD_VECTOR) 202098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune break; 202198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 202298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewArgs[19] = { 202398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(0), 202498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(1), 202598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(2), 202698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(3), 202798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(4), 202898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(5), 202998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(6), 203098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(7), 203198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(8), 203298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(9), 203398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(10), 203498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(11), 203598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(12), 203698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(13), 203798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(14), 203898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(15), 203998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(16), 204098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(17), 204198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(18), 204298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 204398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG); 204498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, SDLoc(N), N->getVTList(), 2045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewArgs); 204698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 2047f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2048dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 2050f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 2051fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2052fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeunestatic bool 2053fe7831861432d71de47ce502e799fb7264b9f24cVincent LejeuneFoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg, 2054f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Abs, SDValue &Sel, SDValue &Imm, SelectionDAG &DAG) { 2055fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune const R600InstrInfo *TII = 2056fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune static_cast<const R600InstrInfo *>(DAG.getTarget().getInstrInfo()); 2057fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Src.isMachineOpcode()) 2058fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2059fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune switch (Src.getMachineOpcode()) { 2060fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::FNEG_R600: 2061fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Neg.getNode()) 2062fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2063fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = Src.getOperand(0); 2064fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Neg = DAG.getTargetConstant(1, MVT::i32); 2065fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2066fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::FABS_R600: 2067fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Abs.getNode()) 2068fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2069fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = Src.getOperand(0); 2070fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Abs = DAG.getTargetConstant(1, MVT::i32); 2071fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2072fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::CONST_COPY: { 2073fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune unsigned Opcode = ParentNode->getMachineOpcode(); 2074fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2075fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2076fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Sel.getNode()) 2077fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2078fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2079fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue CstOffset = Src.getOperand(0); 2080fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (ParentNode->getValueType(0).isVector()) 2081fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2082fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2083fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune // Gather constants values 2084fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SrcIndices[] = { 2085fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2086fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2087fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2), 2088fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2089fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2090fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2091fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2092fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2093fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2094fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2095fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2096fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2097fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune std::vector<unsigned> Consts; 2098dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int OtherSrcIdx : SrcIndices) { 2099fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); 2100fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OtherSrcIdx < 0 || OtherSelIdx < 0) 2101fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune continue; 2102fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (HasDst) { 2103fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune OtherSrcIdx--; 2104fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune OtherSelIdx--; 2105fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2106fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (RegisterSDNode *Reg = 2107fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { 2108fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (Reg->getReg() == AMDGPU::ALU_CONST) { 2109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Cst 2110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx)); 2111fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Consts.push_back(Cst->getZExtValue()); 2112fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2113fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2114fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2115fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset); 2117fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Consts.push_back(Cst->getZExtValue()); 2118fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!TII->fitsConstReadLimitations(Consts)) { 2119fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2120fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2121fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2122fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Sel = CstOffset; 2123fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = DAG.getRegister(AMDGPU::ALU_CONST, MVT::f32); 2124fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2125fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2126f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune case AMDGPU::MOV_IMM_I32: 2127f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune case AMDGPU::MOV_IMM_F32: { 2128f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune unsigned ImmReg = AMDGPU::ALU_LITERAL_X; 2129f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune uint64_t ImmValue = 0; 2130f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2131f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2132f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (Src.getMachineOpcode() == AMDGPU::MOV_IMM_F32) { 2133f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantFPSDNode *FPC = dyn_cast<ConstantFPSDNode>(Src.getOperand(0)); 2134f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune float FloatValue = FPC->getValueAPF().convertToFloat(); 2135f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FloatValue == 0.0) { 2136f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ZERO; 2137f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (FloatValue == 0.5) { 2138f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::HALF; 2139f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (FloatValue == 1.0) { 2140f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ONE; 2141f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2142f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); 2143f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2144f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2145f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src.getOperand(0)); 2146f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune uint64_t Value = C->getZExtValue(); 2147f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (Value == 0) { 2148f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ZERO; 2149f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (Value == 1) { 2150f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ONE_INT; 2151f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2152f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmValue = Value; 2153f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2154f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2155f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2156f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // Check that we aren't already using an immediate. 2157f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // XXX: It's possible for an instruction to have more than one 2158f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // immediate operand, but this is not supported yet. 2159f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (ImmReg == AMDGPU::ALU_LITERAL_X) { 2160f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (!Imm.getNode()) 2161f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return false; 2162f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantSDNode *C = dyn_cast<ConstantSDNode>(Imm); 2163f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune assert(C); 2164f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (C->getZExtValue()) 2165f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return false; 2166f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune Imm = DAG.getTargetConstant(ImmValue, MVT::i32); 2167f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2168f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune Src = DAG.getRegister(ImmReg, MVT::i32); 2169f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return true; 2170f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2171fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune default: 2172fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2173fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2174fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune} 2175fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2176fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2177fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune/// \brief Fold the instructions after selecting them 2178fe7831861432d71de47ce502e799fb7264b9f24cVincent LejeuneSDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node, 2179fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelectionDAG &DAG) const { 2180fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune const R600InstrInfo *TII = 2181fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune static_cast<const R600InstrInfo *>(DAG.getTarget().getInstrInfo()); 2182fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Node->isMachineOpcode()) 2183fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2184fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune unsigned Opcode = Node->getMachineOpcode(); 2185fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue FakeOp; 2186fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2187fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune std::vector<SDValue> Ops; 2188cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (const SDUse &I : Node->ops()) 2189cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Ops.push_back(I); 2190fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2191fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (Opcode == AMDGPU::DOT_4) { 2192fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OperandIdx[] = { 2193fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2194fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2195fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2196fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2197fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2198fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2199fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2200fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2201661bd3df7518a3d984dada66473602a0401618baNAKAMURA Takumi }; 2202fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int NegIdx[] = { 2203fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_X), 2204fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Y), 2205fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Z), 2206fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_W), 2207fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_X), 2208fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Y), 2209fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Z), 2210fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_W) 2211fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2212fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int AbsIdx[] = { 2213fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_X), 2214fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Y), 2215fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Z), 2216fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_W), 2217fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_X), 2218fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Y), 2219fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Z), 2220fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_W) 2221fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2222fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune for (unsigned i = 0; i < 8; i++) { 2223fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OperandIdx[i] < 0) 2224fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2225fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Src = Ops[OperandIdx[i] - 1]; 2226fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Neg = Ops[NegIdx[i] - 1]; 2227fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Abs = Ops[AbsIdx[i] - 1]; 2228fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2229fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2230fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (HasDst) 2231fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelIdx--; 2232fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2233f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG)) 2234f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2235f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2236f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (Opcode == AMDGPU::REG_SEQUENCE) { 2237f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { 2238f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Src = Ops[i]; 2239f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG)) 2240fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2241fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 22425251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune } else if (Opcode == AMDGPU::CLAMP_R600) { 22435251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune SDValue Src = Node->getOperand(0); 22445251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune if (!Src.isMachineOpcode() || 22455251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune !TII->hasInstrModifiers(Src.getMachineOpcode())) 22465251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune return Node; 22475251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune int ClampIdx = TII->getOperandIdx(Src.getMachineOpcode(), 22485251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune AMDGPU::OpName::clamp); 22495251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune if (ClampIdx < 0) 22505251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune return Node; 22515251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune std::vector<SDValue> Ops; 22525251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune unsigned NumOp = Src.getNumOperands(); 22535251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune for(unsigned i = 0; i < NumOp; ++i) 2254661bd3df7518a3d984dada66473602a0401618baNAKAMURA Takumi Ops.push_back(Src.getOperand(i)); 22555251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune Ops[ClampIdx - 1] = DAG.getTargetConstant(1, MVT::i32); 22565251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune return DAG.getMachineNode(Src.getMachineOpcode(), SDLoc(Node), 22575251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune Node->getVTList(), Ops); 2258fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } else { 2259fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!TII->hasInstrModifiers(Opcode)) 2260fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2261fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OperandIdx[] = { 2262fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2263fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2264fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2) 2265fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2266fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int NegIdx[] = { 2267fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg), 2268fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg), 2269fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2_neg) 2270fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2271fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int AbsIdx[] = { 2272fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs), 2273fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs), 2274fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune -1 2275fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2276fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune for (unsigned i = 0; i < 3; i++) { 2277fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OperandIdx[i] < 0) 2278fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2279fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Src = Ops[OperandIdx[i] - 1]; 2280fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Neg = Ops[NegIdx[i] - 1]; 2281fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue FakeAbs; 2282fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; 2283fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2284fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2285f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune int ImmIdx = TII->getOperandIdx(Opcode, AMDGPU::OpName::literal); 2286f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (HasDst) { 2287fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelIdx--; 2288f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmIdx--; 2289f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2290fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2291f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Imm = Ops[ImmIdx]; 2292f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG)) 2293fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2294fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2295fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2296fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2297fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2298fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune} 2299