1f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===// 2f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 3f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// The LLVM Compiler Infrastructure 4f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 5f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// This file is distributed under the University of Illinois Open Source 6f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// License. See LICENSE.TXT for details. 7f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 8f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 9f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 10f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \file 11f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief Custom DAG lowering for R600 12f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 13f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 14f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 15f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600ISelLowering.h" 16c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "AMDGPUFrameLowering.h" 17c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "AMDGPUIntrinsicInfo.h" 18c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "AMDGPUSubtarget.h" 19f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600Defines.h" 20f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600InstrInfo.h" 21f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "R600MachineFunctionInfo.h" 2237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Analysis/ValueTracking.h" 23f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard#include "llvm/CodeGen/CallingConvLower.h" 24c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard#include "llvm/CodeGen/MachineFrameInfo.h" 25f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineInstrBuilder.h" 26f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 27f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 280b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Argument.h" 290b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 30f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 31f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardusing namespace llvm; 32f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 33ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesR600TargetLowering::R600TargetLowering(TargetMachine &TM, 34ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AMDGPUSubtarget &STI) 35ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : AMDGPUTargetLowering(TM, STI), Gen(STI.getGeneration()) { 36f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::v4f32, &AMDGPU::R600_Reg128RegClass); 37f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::f32, &AMDGPU::R600_Reg32RegClass); 38f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::v4i32, &AMDGPU::R600_Reg128RegClass); 39f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard addRegisterClass(MVT::i32, &AMDGPU::R600_Reg32RegClass); 40692ee102ebef535d311c35d53457028083e5c5beTom Stellard addRegisterClass(MVT::v2f32, &AMDGPU::R600_Reg64RegClass); 41692ee102ebef535d311c35d53457028083e5c5beTom Stellard addRegisterClass(MVT::v2i32, &AMDGPU::R600_Reg64RegClass); 42692ee102ebef535d311c35d53457028083e5c5beTom Stellard 43ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines computeRegisterProperties(STI.getRegisterInfo()); 44f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 459c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard // Set condition code actions 469c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETO, MVT::f32, Expand); 479c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 4812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLT, MVT::f32, Expand); 499c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETLE, MVT::f32, Expand); 5012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETOLT, MVT::f32, Expand); 5112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 529c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 539c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 549c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); 559c598cfebcc3387676995873e65ae4fed96b3edcTom Stellard setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 5612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 5712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULE, MVT::f32, Expand); 5812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 5912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLE, MVT::i32, Expand); 6012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETLT, MVT::i32, Expand); 6112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULE, MVT::i32, Expand); 6212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard setCondCodeAction(ISD::SETULT, MVT::i32, Expand); 6312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 64c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune setOperationAction(ISD::FCOS, MVT::f32, Custom); 65c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune setOperationAction(ISD::FSIN, MVT::f32, Custom); 66c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune 67f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SETCC, MVT::v4i32, Expand); 68692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::SETCC, MVT::v2i32, Expand); 69f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 70d41650590669bf561d8f3bcae1204f11354954dcTom Stellard setOperationAction(ISD::BR_CC, MVT::i32, Expand); 71d41650590669bf561d8f3bcae1204f11354954dcTom Stellard setOperationAction(ISD::BR_CC, MVT::f32, Expand); 72c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::BRCOND, MVT::Other, Custom); 73f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 74f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FSUB, MVT::f32, Expand); 75f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 76f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 77f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 78f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i1, Custom); 79f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 80f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 81f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 82f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 839c6b0b0ccebf9d9bf0f357a1c72ef941c5bbb2b2Tom Stellard setOperationAction(ISD::SETCC, MVT::i32, Expand); 849c6b0b0ccebf9d9bf0f357a1c72ef941c5bbb2b2Tom Stellard setOperationAction(ISD::SETCC, MVT::f32, Expand); 85f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom); 8637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 8737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 88f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 89756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::i32, Expand); 90756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::f32, Expand); 91756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::v2i32, Expand); 92756f382ac116d1d935fe5c01f2c07c19c0aac77aTom Stellard setOperationAction(ISD::SELECT, MVT::v4i32, Expand); 93dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // ADD, SUB overflow. 956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // TODO: turn these into Legal? 966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget->hasCARRY()) 976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::UADDO, MVT::i32, Custom); 986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Subtarget->hasBORROW()) 1006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar setOperationAction(ISD::USUBO, MVT::i32, Custom); 1016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Expand sign extension of vectors 103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand); 107dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand); 108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand); 112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand); 113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->hasBFE()) 115dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand); 117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand); 118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 119dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand); 121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand); 122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand); 124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 126c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Legalize loads and stores to the private address space. 127c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::LOAD, MVT::i32, Custom); 128692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::LOAD, MVT::v2i32, Custom); 129c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 13029f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault 13129f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address 13229f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault // spaces, so it is custom lowered to handle those where it isn't. 133ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::integer_valuetypes()) { 134ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 135ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Custom); 136ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Custom); 137ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 138ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 139ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Custom); 140ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Custom); 141ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 142ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 143ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Custom); 144ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Custom); 145ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 14629f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault 147c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::STORE, MVT::i8, Custom); 148f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::i32, Custom); 149692ee102ebef535d311c35d53457028083e5c5beTom Stellard setOperationAction(ISD::STORE, MVT::v2i32, Custom); 150f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::v4i32, Custom); 151ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard setTruncStoreAction(MVT::i32, MVT::i8, Custom); 152ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard setTruncStoreAction(MVT::i32, MVT::i16, Custom); 153f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1549f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setOperationAction(ISD::LOAD, MVT::i32, Custom); 1559f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setOperationAction(ISD::LOAD, MVT::v4i32, Custom); 156c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 157c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 158c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom); 159c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom); 160c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 161c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 162c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 163c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom); 164c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom); 165c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 166c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 167c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 168f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setTargetDAGCombine(ISD::FP_ROUND); 1691234c9be42b4ebd4b398df461123205dccf3706cTom Stellard setTargetDAGCombine(ISD::FP_TO_SINT); 1709f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 1711234c9be42b4ebd4b398df461123205dccf3706cTom Stellard setTargetDAGCombine(ISD::SELECT_CC); 17215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 173f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 174c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 175c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // to be Legal/Custom in order to avoid library calls. 176c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 177c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 178c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 180a3e39dc7055486cbf514ccd868cfabc69d7f6f4eMichel Danzer setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 181a3e39dc7055486cbf514ccd868cfabc69d7f6f4eMichel Danzer 182c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 183c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (MVT VT : ScalarIntVTs) { 184c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::ADDC, VT, Expand); 185c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::SUBC, VT, Expand); 186c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::ADDE, VT, Expand); 187c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines setOperationAction(ISD::SUBE, VT, Expand); 188c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 189c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 190f15dfe4eb48e8e2ff02a30bc8ba9112108f9b83dTom Stellard setSchedulingPreference(Sched::Source); 191f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 192f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 193cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainarstatic inline bool isEOP(MachineBasicBlock::iterator I) { 194cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return std::next(I)->getOpcode() == AMDGPU::RETURN; 195cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar} 196cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 197f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardMachineBasicBlock * R600TargetLowering::EmitInstrWithCustomInserter( 198f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr * MI, MachineBasicBlock * BB) const { 199f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineFunction * MF = BB->getParent(); 200f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineRegisterInfo &MRI = MF->getRegInfo(); 201f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineBasicBlock::iterator I = *MI; 202b5632b5b456db647b42239cbd4d8b58c82290c4eBill Wendling const R600InstrInfo *TII = 203ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const R600InstrInfo *>(Subtarget->getInstrInfo()); 204f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 205f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (MI->getOpcode()) { 206a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard default: 20719a99df130f5747da950faf4ca5170d71f05594cTom Stellard // Replace LDS_*_RET instruction that don't have any uses with the 20819a99df130f5747da950faf4ca5170d71f05594cTom Stellard // equivalent LDS_*_NORET instruction. 20919a99df130f5747da950faf4ca5170d71f05594cTom Stellard if (TII->isLDSRetInstr(MI->getOpcode())) { 21079916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard int DstIdx = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst); 21179916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard assert(DstIdx != -1); 21279916948e1fd176a3898b596b679cc9dba3d40a8Tom Stellard MachineInstrBuilder NewMI; 21337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add 21437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // LDS_1A2D support and remove this special case. 21537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!MRI.use_empty(MI->getOperand(DstIdx).getReg()) || 21637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MI->getOpcode() == AMDGPU::LDS_CMPST_RET) 21719a99df130f5747da950faf4ca5170d71f05594cTom Stellard return BB; 21819a99df130f5747da950faf4ca5170d71f05594cTom Stellard 21919a99df130f5747da950faf4ca5170d71f05594cTom Stellard NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), 22019a99df130f5747da950faf4ca5170d71f05594cTom Stellard TII->get(AMDGPU::getLDSNoRetOp(MI->getOpcode()))); 221a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard for (unsigned i = 1, e = MI->getNumOperands(); i < e; ++i) { 222a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard NewMI.addOperand(MI->getOperand(i)); 223a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } 224a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } else { 225a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 226a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard } 227a01cdea9c660dc8b295782a4ab560d0039ff7571Tom Stellard break; 228f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::CLAMP_R600: { 229f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 230f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 231f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 232f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 233f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_CLAMP); 234f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 235f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 236f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 237f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::FABS_R600: { 238f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 239f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 240f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 241f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 242f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_ABS); 243f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 244f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 245f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 246f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::FNEG_R600: { 247f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I, 248f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::MOV, 249f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(0).getReg(), 250f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getReg()); 251f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_NEG); 252f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 253f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 254f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 255f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MASK_WRITE: { 256f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned maskedRegister = MI->getOperand(0).getReg(); 257f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(TargetRegisterInfo::isVirtualRegister(maskedRegister)); 258f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr * defInstr = MRI.getVRegDef(maskedRegister); 259f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(defInstr, 0, MO_FLAG_MASK); 260f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 261f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 262f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 263f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MOV_IMM_F32: 264f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 265f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getFPImm()->getValueAPF() 266f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .bitcastToAPInt().getZExtValue()); 267f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 268f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::MOV_IMM_I32: 269f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(), 270f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->getOperand(1).getImm()); 271f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 272d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune case AMDGPU::CONST_COPY: { 273d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV, 274d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MI->getOperand(0).getReg(), AMDGPU::ALU_CONST); 2755e48a0e9ae2365a130dd1ec2e0b4beb337ab79e0Tom Stellard TII->setImmOperand(NewMI, AMDGPU::OpName::src0_sel, 276d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune MI->getOperand(1).getImm()); 277d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune break; 278d4c3e566922e04296c29cc4a1695e06a2b53bcb7Vincent Lejeune } 279f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 280f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::RAT_WRITE_CACHELESS_32_eg: 281692ee102ebef535d311c35d53457028083e5c5beTom Stellard case AMDGPU::RAT_WRITE_CACHELESS_64_eg: 282f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::RAT_WRITE_CACHELESS_128_eg: { 283f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 284f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 285f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 286cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar .addImm(isEOP(I)); // Set End of program bit 287cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar break; 288cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar } 289cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case AMDGPU::RAT_STORE_TYPED_eg: { 290cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 291cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar .addOperand(MI->getOperand(0)) 292cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar .addOperand(MI->getOperand(1)) 293cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar .addOperand(MI->getOperand(2)) 294cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar .addImm(isEOP(I)); // Set End of program bit 295f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 296f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 297f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 298f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::TXD: { 299f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 300f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 301d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &RID = MI->getOperand(4); 302d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &SID = MI->getOperand(5); 303d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureId = MI->getOperand(6).getImm(); 304d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 305d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 306d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 307d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (TextureId) { 308d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 5: // Rect 309d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 310d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 311d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 6: // Shadow1D 312d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 313d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 314d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 7: // Shadow2D 315d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 316d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 317d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 8: // ShadowRect 318d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 319d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 320d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 321d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 9: // 1DArray 322d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 323d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 324d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 325d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 10: // 2DArray 326d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 327d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 328d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 11: // Shadow1DArray 329d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 330d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 331d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 332d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 12: // Shadow2DArray 333d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 334d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 335d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 336f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 337f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 338d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 339d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 340d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 341d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 342d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 343d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 344d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 345d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 346d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 347d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 348d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 349d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 350d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 351d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 352d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 353d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 354d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 355f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 356f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 357d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 358d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 359d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 360d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 361d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 362d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 363d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 364d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 365d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 366d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 367d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 368d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 369d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 370d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 371d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 372d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 373d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 374f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_G)) 375f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 376f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 377d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 378d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 379d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 380d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 381d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 382d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 383d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 384d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 385d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 386d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 387d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 388d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 389d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 390d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 391d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 392d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 393d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW) 394f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T0, RegState::Implicit) 395f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T1, RegState::Implicit); 396f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 397f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 398f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 399f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::TXD_SHADOW: { 400f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 401f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass); 402d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &RID = MI->getOperand(4); 403d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune MachineOperand &SID = MI->getOperand(5); 404d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureId = MI->getOperand(6).getImm(); 405d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned SrcX = 0, SrcY = 1, SrcZ = 2, SrcW = 3; 406d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned CTX = 1, CTY = 1, CTZ = 1, CTW = 1; 407d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 408d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (TextureId) { 409d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 5: // Rect 410d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 411d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 412d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 6: // Shadow1D 413d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 414d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 415d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 7: // Shadow2D 416d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 417d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 418d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 8: // ShadowRect 419d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTX = CTY = 0; 420d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcW = SrcZ; 421d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 422d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 9: // 1DArray 423d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 424d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 425d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 426d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 10: // 2DArray 427d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 428d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 429d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 11: // Shadow1DArray 430d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SrcZ = SrcY; 431d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 432d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 433d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case 12: // Shadow2DArray 434d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune CTZ = 0; 435d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 436d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 437f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 438f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0) 439f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 440d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 441d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 442d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 443d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 444d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 445d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 446d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 447d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 448d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 449d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 450d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 451d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 452d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 453d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 454d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 455d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 456d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 457f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1) 458f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 459d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 460d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 461d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 462d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 463d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 464d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 465d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 466d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 467d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 468d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 469d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 470d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 471d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 472d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 473d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 474d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 475d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW); 476f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_C_G)) 477f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 478f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 479d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcX) 480d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcY) 481d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcZ) 482d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(SrcW) 483d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 484d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 485d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 486d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(0) 487d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(1) 488d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(2) 489d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(3) 490d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(RID) 491d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addOperand(SID) 492d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTX) 493d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTY) 494d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTZ) 495d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune .addImm(CTW) 496f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T0, RegState::Implicit) 497f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(T1, RegState::Implicit); 498f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 499f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 500f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 501f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH: 502f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP)) 503fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune .addOperand(MI->getOperand(0)); 504f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 505f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 506f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH_COND_f32: { 507f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = 508f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 509f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::PREDICATE_BIT) 510f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 511f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(OPCODE_IS_NOT_ZERO) 512f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(0); // Flags 513f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 514fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 515f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 516f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 517f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 518f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 519f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 520f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::BRANCH_COND_i32: { 521f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineInstr *NewMI = 522f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X), 523f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::PREDICATE_BIT) 524f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 525f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(OPCODE_IS_NOT_ZERO_INT) 526f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(0); // Flags 527f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TII->addFlag(NewMI, 0, MO_FLAG_PUSH); 528fd49dac48fee6da580157515dec55ed2f2d8f2b3Vincent Lejeune BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP_COND)) 529f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 530f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill); 531f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 532f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 533f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 534f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::EG_ExportSwz: 535f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPU::R600_ExportSwz: { 536254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard // Instruction is left unmodified if its not the last one of its type 537254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard bool isLastInstructionOfItsType = true; 538254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard unsigned InstExportType = MI->getOperand(1).getImm(); 53936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (MachineBasicBlock::iterator NextExportInst = std::next(I), 540254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard EndBlock = BB->end(); NextExportInst != EndBlock; 54136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NextExportInst = std::next(NextExportInst)) { 542254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (NextExportInst->getOpcode() == AMDGPU::EG_ExportSwz || 543254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard NextExportInst->getOpcode() == AMDGPU::R600_ExportSwz) { 544254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard unsigned CurrentInstExportType = NextExportInst->getOperand(1) 545254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard .getImm(); 546254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (CurrentInstExportType == InstExportType) { 547254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard isLastInstructionOfItsType = false; 548254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard break; 549254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 550254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 551254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard } 552cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar bool EOP = isEOP(I); 553254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard if (!EOP && !isLastInstructionOfItsType) 554f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return BB; 555f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned CfInst = (MI->getOpcode() == AMDGPU::EG_ExportSwz)? 84 : 40; 556f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode())) 557f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(0)) 558f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(1)) 559f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(2)) 560f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(3)) 561f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(4)) 562f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(5)) 563f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addOperand(MI->getOperand(6)) 564f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard .addImm(CfInst) 565254a83e46c0ffb08c5c77d99f64d6e86db550c6fTom Stellard .addImm(EOP); 566f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 567f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 568a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen case AMDGPU::RETURN: { 569a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen // RETURN instructions must have the live-out registers as implicit uses, 570a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen // otherwise they appear dead. 571a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen R600MachineFunctionInfo *MFI = MF->getInfo<R600MachineFunctionInfo>(); 572a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MachineInstrBuilder MIB(*MF, MI); 573a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen for (unsigned i = 0, e = MFI->LiveOuts.size(); i != e; ++i) 574a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MIB.addReg(MFI->LiveOuts[i], RegState::Implicit); 575a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen return BB; 576a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen } 577f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 578f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 579f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MI->eraseFromParent(); 580f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return BB; 581f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 582f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 583f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 584f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Custom DAG Lowering Operations 585f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 586f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 587f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 588e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 589e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 590f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Op.getOpcode()) { 591f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 592c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 593c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 594c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG); 595c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SRA_PARTS: 596c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG); 5976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY); 5986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW); 599c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FCOS: 600c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FSIN: return LowerTrig(Op, DAG); 601f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 602f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::STORE: return LowerSTORE(Op, DAG); 603c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::LOAD: { 604c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Result = LowerLOAD(Op, DAG); 605c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines assert((!Result.getNode() || 606c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Result.getNode()->getNumValues() == 2) && 607c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines "Load should return a value and a chain"); 608c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Result; 609c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 610c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 611c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::BRCOND: return LowerBRCOND(Op, DAG); 612e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG); 613f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_VOID: { 614f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain = Op.getOperand(0); 615f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = 616f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 617f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (IntrinsicID) { 618f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_store_output: { 619f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 620f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex); 621a499d2bcef0c1001c60d752d356e50eed2402ca8Jakob Stoklund Olesen MFI->LiveOuts.push_back(Reg); 622ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getCopyToReg(Chain, SDLoc(Op), Reg, Op.getOperand(2)); 623f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 624abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune case AMDGPUIntrinsic::R600_store_swizzle: { 6256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 626abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune const SDValue Args[8] = { 627abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Chain, 628abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(2), // Export Value 629abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(3), // ArrayBase 630abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune Op.getOperand(4), // Type 6316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32), // SWZ_X 6326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32), // SWZ_Y 6336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32), // SWZ_Z 6346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, MVT::i32) // SWZ_W 635abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune }; 6366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(AMDGPUISD::EXPORT, DL, Op.getValueType(), Args); 637f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 638f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 639f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // default for switch(IntrinsicID) 640f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: break; 641f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 642f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode()) 643f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 644f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 645f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_WO_CHAIN: { 646f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = 647f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 648f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 649ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 650f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch(IntrinsicID) { 651f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 652837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune case AMDGPUIntrinsic::R600_load_input: { 653837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 654837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex); 655837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 656837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineRegisterInfo &MRI = MF.getRegInfo(); 657837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(Reg); 658837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return DAG.getCopyFromReg(DAG.getEntryNode(), 659837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), Reg, VT); 660837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 661837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune 662837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune case AMDGPUIntrinsic::R600_interp_input: { 663837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 664837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune int ijb = cast<ConstantSDNode>(Op.getOperand(2))->getSExtValue(); 665837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineSDNode *interp; 666837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune if (ijb < 0) { 667ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const R600InstrInfo *TII = 668ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const R600InstrInfo *>(Subtarget->getInstrInfo()); 669837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_VEC_LOAD, DL, 6706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::v4f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32)); 671837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return DAG.getTargetExtractSubreg( 672837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune TII->getRegisterInfo().getSubRegFromChannel(slot % 4), 673837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune DL, MVT::f32, SDValue(interp, 0)); 674837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 675837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 676837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MachineRegisterInfo &MRI = MF.getRegInfo(); 677837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned RegisterI = AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb); 678837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune unsigned RegisterJ = AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb + 1); 679837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(RegisterI); 680837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune MRI.addLiveIn(RegisterJ); 681837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDValue RegisterINode = DAG.getCopyFromReg(DAG.getEntryNode(), 682837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), RegisterI, MVT::f32); 683837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDValue RegisterJNode = DAG.getCopyFromReg(DAG.getEntryNode(), 684837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune SDLoc(DAG.getEntryNode()), RegisterJ, MVT::f32); 685837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune 686837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune if (slot % 4 < 2) 687837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 6886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32), 689837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune RegisterJNode, RegisterINode); 690837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune else 691837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 6926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32), 693837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune RegisterJNode, RegisterINode); 694837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune return SDValue(interp, slot % 2); 695837dd95d6c8cb4f23df4e54eac027eb289991629Vincent Lejeune } 69670a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune case AMDGPUIntrinsic::R600_interp_xy: 69770a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune case AMDGPUIntrinsic::R600_interp_zw: { 698f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 69929b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard MachineSDNode *interp; 70070a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue RegisterINode = Op.getOperand(2); 70170a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue RegisterJNode = Op.getOperand(3); 70229b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard 70370a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune if (IntrinsicID == AMDGPUIntrinsic::R600_interp_xy) 70429b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL, 7056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 7060962e147a439785279c3665379189017e980e0ccVincent Lejeune RegisterJNode, RegisterINode); 70729b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard else 70829b15a378045762ce09642ab9dd741ece41f59a3Tom Stellard interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL, 7096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32), 7100962e147a439785279c3665379189017e980e0ccVincent Lejeune RegisterJNode, RegisterINode); 71170a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, 71270a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue(interp, 0), SDValue(interp, 1)); 713f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 714d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_tex: 715d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_texc: 716d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txl: 717d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txlc: 718d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txb: 719d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txbc: 720d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txf: 721d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txq: 722d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddx: 723a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ddy: 724a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ldptr: { 725d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune unsigned TextureOp; 726d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune switch (IntrinsicID) { 727d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_tex: 728d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 0; 729d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 730d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_texc: 731d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 1; 732d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 733d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txl: 734d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 2; 735d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 736d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txlc: 737d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 3; 738d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 739d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txb: 740d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 4; 741d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 742d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txbc: 743d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 5; 744d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 745d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txf: 746d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 6; 747d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 748d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_txq: 749d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 7; 750d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 751d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddx: 752d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 8; 753d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 754d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune case AMDGPUIntrinsic::R600_ddy: 755d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune TextureOp = 9; 756d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune break; 757a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune case AMDGPUIntrinsic::R600_ldptr: 758a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune TextureOp = 10; 759a2f1317f09ac6b4a7239b033fabd216d71b77629Vincent Lejeune break; 760d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune default: 761d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune llvm_unreachable("Unknow Texture Operation"); 762d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 763d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune 764d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune SDValue TexArgs[19] = { 7656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(TextureOp, DL, MVT::i32), 766d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(1), 7676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32), 7686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32), 7696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32), 7706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, MVT::i32), 771d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(2), 772d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(3), 773d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(4), 7746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32), 7756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32), 7766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32), 7776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, MVT::i32), 778d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(5), 779d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(6), 780d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(7), 781d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(8), 782d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(9), 783d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune Op.getOperand(10) 784d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune }; 785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs); 786d3293b49f9c7af741d2edd3062499fb50db0e89bVincent Lejeune } 7874ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune case AMDGPUIntrinsic::AMDGPU_dp4: { 7884ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune SDValue Args[8] = { 7894ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)), 7914ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)), 7934ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32)), 7954ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 7966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(1, DL, MVT::i32)), 7974ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 7986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32)), 7994ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 8006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32)), 8014ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1), 8026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, MVT::i32)), 8034ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2), 8046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, MVT::i32)) 8054ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune }; 806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args); 8074ed9917147b1d1f2616f7c941bbe6999b979f510Vincent Lejeune } 808f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 8097849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_x: 810f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 0); 8117849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_y: 812f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 1); 8137849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_ngroups_z: 814f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 2); 8157849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_x: 816f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 3); 8177849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_y: 818f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 4); 8197849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_global_size_z: 820f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 5); 8217849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_x: 822f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 6); 8237849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_y: 824f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 7); 8257849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_local_size_z: 826f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerImplicitParameter(DAG, VT, DL, 8); 827f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 828cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case Intrinsic::AMDGPU_read_workdim: { 829cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar uint32_t ByteOffset = getImplicitParameterOffset(MFI, GRID_DIM); 830cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return LowerImplicitParameter(DAG, VT, DL, ByteOffset / 4); 831cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar } 83237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 8337849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_x: 834f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 835f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_X, VT); 8367849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_y: 837f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 838f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_Y, VT); 8397849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tgid_z: 840f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 841f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T1_Z, VT); 8427849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_x: 843f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 844f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_X, VT); 8457849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_y: 846f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 847f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_Y, VT); 8487849728485570f34bb100baef613a80d84450b08NAKAMURA Takumi case Intrinsic::r600_read_tidig_z: 849f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, 850f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AMDGPU::T0_Z, VT); 851c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case Intrinsic::AMDGPU_rsq: 852c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // XXX - I'm assuming SI's RSQ_LEGACY matches R600's behavior. 853c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 8544c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 8554c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AMDGPUIntrinsic::AMDGPU_fract: 8564c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AMDGPUIntrinsic::AMDIL_fraction: // Legacy name. 8574c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 858f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 859f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode()) 860f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 861f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 862f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } // end switch(Op.getOpcode()) 863f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return SDValue(); 864f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 865f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 866f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardvoid R600TargetLowering::ReplaceNodeResults(SDNode *N, 867f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVectorImpl<SDValue> &Results, 868f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 869f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (N->getOpcode()) { 87036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 87136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG); 87236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 87337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::FP_TO_UINT: 87437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (N->getValueType(0) == MVT::i1) { 87537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Results.push_back(LowerFPTOUINT(N->getOperand(0), DAG)); 87637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return; 87737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 87837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Fall-through. Since we don't care about out of bounds values 87937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we can use FP_TO_SINT for uints too. The DAGLegalizer code for uint 88037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // considers some extra cases which are not necessary here. 88137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::FP_TO_SINT: { 88237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Result; 88337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (expandFP_TO_SINT(N, Result, DAG)) 88437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Results.push_back(Result); 8859f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return; 88637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 887c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SDIVREM: { 888c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Op = SDValue(N, 1); 889c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue RES = LowerSDIVREM(Op, DAG); 890c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Results.push_back(RES); 891c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Results.push_back(RES.getValue(1)); 892c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines break; 893c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 894c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::UDIVREM: { 895c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Op = SDValue(N, 0); 89637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LowerUDIVREM64(Op, DAG, Results); 897c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines break; 898f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 899c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 900c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 901c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 902c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG, 903c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Vector) const { 904c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 905c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Vector); 906c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VecVT = Vector.getValueType(); 907c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT EltVT = VecVT.getVectorElementType(); 908c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<SDValue, 8> Args; 909c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 910c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (unsigned i = 0, e = VecVT.getVectorNumElements(); 911c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines i != e; ++i) { 912cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar Args.push_back(DAG.getNode( 913cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector, 914cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar DAG.getConstant(i, DL, getVectorIdxTy(DAG.getDataLayout())))); 915c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 916c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 917c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args); 918c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 919c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 920c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 921c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SelectionDAG &DAG) const { 922c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 923c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 924c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Vector = Op.getOperand(0); 925c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Index = Op.getOperand(1); 926c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 927c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isa<ConstantSDNode>(Index) || 928c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 929c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Op; 930c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 931c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector = vectorToVerticalVector(DAG, Vector); 932c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), 933c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector, Index); 934c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 935c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 936c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 937c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SelectionDAG &DAG) const { 938c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 939c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Vector = Op.getOperand(0); 940c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Value = Op.getOperand(1); 941c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Index = Op.getOperand(2); 942c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 943c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isa<ConstantSDNode>(Index) || 944c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR) 945c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Op; 946c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 947c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector = vectorToVerticalVector(DAG, Vector); 948c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(), 949c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Vector, Value, Index); 950c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return vectorToVerticalVector(DAG, Insert); 951f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 952f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 953c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent LejeuneSDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const { 954c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // On hw >= R700, COS/SIN input must be between -1. and 1. 955c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) 956c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune EVT VT = Op.getValueType(); 957c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune SDValue Arg = Op.getOperand(0); 9586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 959cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 960cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // TODO: Should this propagate fast-math-flags? 9616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT, 9626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::FADD, DL, VT, 9636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::FMUL, DL, VT, Arg, 9646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstantFP(0.15915494309, DL, MVT::f32)), 9656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstantFP(0.5, DL, MVT::f32))); 966c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune unsigned TrigNode; 967c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune switch (Op.getOpcode()) { 968c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FCOS: 969c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune TrigNode = AMDGPUISD::COS_HW; 970c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune break; 971c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune case ISD::FSIN: 972c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune TrigNode = AMDGPUISD::SIN_HW; 973c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune break; 974c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune default: 975c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune llvm_unreachable("Wrong trig opcode"); 976c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune } 9776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue TrigVal = DAG.getNode(TrigNode, DL, VT, 9786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::FADD, DL, VT, FractPart, 9796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstantFP(-0.5, DL, MVT::f32))); 980c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune if (Gen >= AMDGPUSubtarget::R700) 981c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune return TrigVal; 982c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune // On R600 hw, COS/SIN input must be between -Pi and Pi. 9836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::FMUL, DL, VT, TrigVal, 9846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstantFP(3.14159265359, DL, MVT::f32)); 985c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune} 986c6f13db656c7649f933c74c4f90c09ff74de52a8Vincent Lejeune 987c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const { 988c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 989c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 990c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 991c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Lo = Op.getOperand(0); 992c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Hi = Op.getOperand(1); 993c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Shift = Op.getOperand(2); 9946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Zero = DAG.getConstant(0, DL, VT); 9956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue One = DAG.getConstant(1, DL, VT); 996c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 9976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 9986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 999c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1000c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1001c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1002c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // The dance around Width1 is necessary for 0 special case. 1003c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Without it the CompShift might be 32, producing incorrect results in 1004c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Overflow. So we do the shift in two steps, the alternative is to 1005c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // add a conditional to filter the special case. 1006c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1007c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift); 1008c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Overflow = DAG.getNode(ISD::SRL, DL, VT, Overflow, One); 1009c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1010c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift); 1011c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines HiSmall = DAG.getNode(ISD::OR, DL, VT, HiSmall, Overflow); 1012c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift); 1013c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1014c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift); 1015c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue LoBig = Zero; 1016c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1017c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1018c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1019c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1020c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1021c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 1022c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1023c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const { 1024c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 1025c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 1026c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1027c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Lo = Op.getOperand(0); 1028c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Hi = Op.getOperand(1); 1029c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Shift = Op.getOperand(2); 10306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Zero = DAG.getConstant(0, DL, VT); 10316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue One = DAG.getConstant(1, DL, VT); 1032c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1033c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const bool SRA = Op.getOpcode() == ISD::SRA_PARTS; 1034c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 10356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT); 10366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); 1037c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width); 1038c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift); 1039c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1040c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // The dance around Width1 is necessary for 0 special case. 1041c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Without it the CompShift might be 32, producing incorrect results in 1042c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Overflow. So we do the shift in two steps, the alternative is to 1043c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // add a conditional to filter the special case. 1044c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1045c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift); 1046c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Overflow = DAG.getNode(ISD::SHL, DL, VT, Overflow, One); 1047c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1048c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift); 1049c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift); 1050c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoSmall = DAG.getNode(ISD::OR, DL, VT, LoSmall, Overflow); 1051c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1052c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift); 1053c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero; 1054c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1055c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Hi = DAG.getSelectCC(DL, Shift, Width, HiSmall, HiBig, ISD::SETULT); 1056c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Lo = DAG.getSelectCC(DL, Shift, Width, LoSmall, LoBig, ISD::SETULT); 1057c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1058c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi); 1059c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 1060c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 10616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga NainarSDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG, 10626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned mainop, unsigned ovf) const { 10636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 10646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT VT = Op.getValueType(); 10656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 10666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Lo = Op.getOperand(0); 10676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Hi = Op.getOperand(1); 10686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 10696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi); 10706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Extend sign. 10716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF, 10726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getValueType(MVT::i1)); 10736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 10746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi); 10756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 10766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF); 10776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 10786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1079f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const { 10806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 1081f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode( 1082f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETCC, 10836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DL, 1084f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MVT::i1, 10856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op, DAG.getConstantFP(0.0f, DL, MVT::f32), 1086f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(ISD::SETNE) 1087f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ); 1088f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1089f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1090f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT, 1091ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, 1092f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned DwordOffset) const { 1093f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned ByteOffset = DwordOffset * 4; 1094f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 1095a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard AMDGPUAS::CONSTANT_BUFFER_0); 1096f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1097f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // We shouldn't be using an offset wider than 16-bits for implicit parameters. 1098f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(isInt<16>(ByteOffset)); 1099f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1100f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getLoad(VT, DL, DAG.getEntryNode(), 11016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR 1102f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachinePointerInfo(ConstantPointerNull::get(PtrType)), 1103f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard false, false, false, 0); 1104f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1105f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1106f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool R600TargetLowering::isZero(SDValue Op) const { 1107f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) { 1108f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Cst->isNullValue(); 1109f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){ 1110f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CstFP->isZero(); 1111f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else { 1112f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 1113f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1114f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1115f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1116f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 1117ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 1118f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 1119f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1120f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue LHS = Op.getOperand(0); 1121f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RHS = Op.getOperand(1); 1122f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue True = Op.getOperand(2); 1123f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue False = Op.getOperand(3); 1124f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue CC = Op.getOperand(4); 1125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Temp; 1126f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1127ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (VT == MVT::f32) { 1128ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); 1129ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue MinMax = CombineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI); 1130ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (MinMax) 1131ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return MinMax; 1132ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1133ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1134f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // LHS and RHS are guaranteed to be the same value type 1135f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT CompareVT = LHS.getValueType(); 1136f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1137f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Check if we can lower this to a native operation. 1138f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 11391454cb86be54a8703fca396752be71c50c805b88Tom Stellard // Try to lower to a SET* instruction: 11401454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 11411454cb86be54a8703fca396752be71c50c805b88Tom Stellard // SET* can match the following patterns: 11421454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 114312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, f32, -1, 0, cc_supported 114412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, f32, 1.0f, 0.0f, cc_supported 114512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, i32, -1, 0, cc_supported 11461454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 11471454cb86be54a8703fca396752be71c50c805b88Tom Stellard 11481454cb86be54a8703fca396752be71c50c805b88Tom Stellard // Move hardware True/False values to the correct operand. 114912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 115012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode InverseCC = 115112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1152bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isHWTrueValue(False) && isHWFalseValue(True)) { 1153bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { 1154bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(False, True); 1155bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard CC = DAG.getCondCode(InverseCC); 1156bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } else { 1157bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC); 1158bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) { 1159bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(False, True); 1160bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard std::swap(LHS, RHS); 1161bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard CC = DAG.getCondCode(SwapInvCC); 1162bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } 1163bbafe422d6f9036b03992ee5eacb5d09644c3267Tom Stellard } 11641454cb86be54a8703fca396752be71c50c805b88Tom Stellard } 11651454cb86be54a8703fca396752be71c50c805b88Tom Stellard 11661454cb86be54a8703fca396752be71c50c805b88Tom Stellard if (isHWTrueValue(True) && isHWFalseValue(False) && 11671454cb86be54a8703fca396752be71c50c805b88Tom Stellard (CompareVT == VT || VT == MVT::i32)) { 11681454cb86be54a8703fca396752be71c50c805b88Tom Stellard // This can be matched by a SET* instruction. 11691454cb86be54a8703fca396752be71c50c805b88Tom Stellard return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC); 11701454cb86be54a8703fca396752be71c50c805b88Tom Stellard } 11711454cb86be54a8703fca396752be71c50c805b88Tom Stellard 1172f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Try to lower to a CND* instruction: 11731454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 11741454cb86be54a8703fca396752be71c50c805b88Tom Stellard // CND* can match the following patterns: 11751454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 117612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, 0.0, f32, f32, cc_supported 117712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc f32, 0.0, i32, i32, cc_supported 117812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, 0, f32, f32, cc_supported 117912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // select_cc i32, 0, i32, i32, cc_supported 11801454cb86be54a8703fca396752be71c50c805b88Tom Stellard // 118112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard 118212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try to move the zero value to the RHS 118312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isZero(LHS)) { 118412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 118512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try swapping the operands 118612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode); 118712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 118812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(LHS, RHS); 118912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CC = DAG.getCondCode(CCSwapped); 119012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } else { 119112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard // Try inverting the conditon and then swapping the operands 119212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT.isInteger()); 119312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CCSwapped = ISD::getSetCCSwappedOperands(CCInv); 119412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { 119512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(True, False); 119612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard std::swap(LHS, RHS); 119712d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard CC = DAG.getCondCode(CCSwapped); 119812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 119912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 120012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard } 120112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (isZero(RHS)) { 120212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard SDValue Cond = LHS; 120312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard SDValue Zero = RHS; 1204f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1205f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (CompareVT != VT) { 1206f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Bitcast True / False to the correct types. This will end up being 1207f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // a nop, but it allows us to define only a single pattern in the 1208f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // .TD files for each CND* instruction rather than having to have 1209f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // one pattern for integer True/False and one for fp True/False 1210f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True); 1211f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False); 1212f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1213f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1214f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (CCOpcode) { 1215f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETONE: 1216f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUNE: 1217f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETNE: 1218f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); 1219f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Temp = True; 1220f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True = False; 1221f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard False = Temp; 1222f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1223f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: 1224f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1225f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1226f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, 1227f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Cond, Zero, 1228f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True, False, 1229f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(CCOpcode)); 1230f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode); 1231f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1232f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1233f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // If we make it this for it means we have no native instructions to handle 1234f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // this SELECT_CC, so we must lower it. 1235f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue HWTrue, HWFalse; 1236f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1237f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (CompareVT == MVT::f32) { 12386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT); 12396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT); 1240f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else if (CompareVT == MVT::i32) { 12416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HWTrue = DAG.getConstant(-1, DL, CompareVT); 12426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HWFalse = DAG.getConstant(0, DL, CompareVT); 1243f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1244f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else { 124536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Unhandled value type in LowerSELECT_CC"); 1246f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1247f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1248f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Lower this unsupported SELECT_CC into a combination of two supported 1249f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // SELECT_CC operations. 1250f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC); 1251f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1252f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::SELECT_CC, DL, VT, 1253f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Cond, HWFalse, 1254f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard True, False, 1255f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getCondCode(ISD::SETNE)); 1256f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1257f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 125836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// LLVM generates byte-addressed pointers. For indirect addressing, we need to 1259c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// convert these pointers to a register index. Each register holds 1260c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// 16 bytes, (4 x 32bit sub-register), but we need to take into account the 1261c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// \p StackWidth, which tells us how many of the 4 sub-registrers will be used 1262c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard/// for indirect addressing. 1263c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom StellardSDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr, 1264c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth, 1265c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SelectionDAG &DAG) const { 1266c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned SRLPad; 1267c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard switch(StackWidth) { 1268c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 1: 1269c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 2; 1270c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1271c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 2: 1272c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 3; 1273c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1274c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 4: 1275c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SRLPad = 4; 1276c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1277c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard default: llvm_unreachable("Invalid stack width"); 1278c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1279c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 12806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Ptr); 12816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr, 12826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(SRLPad, DL, MVT::i32)); 1283c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard} 1284c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1285c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellardvoid R600TargetLowering::getStackAddress(unsigned StackWidth, 1286c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned ElemIdx, 1287c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned &Channel, 1288c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned &PtrIncr) const { 1289c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard switch (StackWidth) { 1290c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard default: 1291c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 1: 1292c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = 0; 1293c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ElemIdx > 0) { 1294c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 1; 1295c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1296c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1297c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1298c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1299c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 2: 1300c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = ElemIdx % 2; 1301c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ElemIdx == 2) { 1302c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 1; 1303c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1304c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1305c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1306c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1307c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard case 4: 1308c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Channel = ElemIdx; 1309c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard PtrIncr = 0; 1310c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard break; 1311c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1312c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard} 1313c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1314f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1315ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 1316f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard StoreSDNode *StoreNode = cast<StoreSDNode>(Op); 1317f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain = Op.getOperand(0); 1318f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Value = Op.getOperand(1); 1319f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Ptr = Op.getOperand(2); 1320f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 13217a0282daeb214f14d75249cc2d90302c44586c4eTom Stellard SDValue Result = AMDGPUTargetLowering::LowerSTORE(Op, DAG); 13224c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard if (Result.getNode()) { 13234c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard return Result; 13244c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard } 13254c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 1326ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS) { 1327ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->isTruncatingStore()) { 1328ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard EVT VT = Value.getValueType(); 13294c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard assert(VT.bitsLE(MVT::i32)); 1330ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard EVT MemVT = StoreNode->getMemoryVT(); 1331ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue MaskConstant; 1332ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (MemVT == MVT::i8) { 13336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32); 1334ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else { 1335ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard assert(MemVT == MVT::i16); 13366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32); 1337ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } 1338ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr, 13396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32)); 1340ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr, 13416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0x00000003, DL, VT)); 1342ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant); 1343ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex, 13446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(3, DL, VT)); 1345ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, Shift); 1346ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, Shift); 1347ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 1348ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // vector instead. 1349ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Src[4] = { 1350ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard ShiftedValue, 13516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32), 13526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32), 1353ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Mask 1354ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard }; 1355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Input = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i32, Src); 1356ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard SDValue Args[3] = { Chain, Input, DWordAddr }; 1357ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL, 1358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op->getVTList(), Args, MemVT, 1359ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard StoreNode->getMemOperand()); 1360ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && 1361ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Value.getValueType().bitsGE(MVT::i32)) { 1362ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard // Convert pointer from byte address to dword address. 1363ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(), 1364ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), 13656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ptr, DAG.getConstant(2, DL, MVT::i32))); 1366ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard 1367ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) { 136836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm_unreachable("Truncated and indexed stores not supported yet"); 1369ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } else { 1370ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); 1371ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard } 1372ec484277dd04399d7b2ea37508e39fc4998bc9a7Tom Stellard return Chain; 1373f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1374f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1375c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1376c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ValueVT = Value.getValueType(); 1377c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1378c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (StoreNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1379c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return SDValue(); 1380c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1381c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 138236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue Ret = AMDGPUTargetLowering::LowerSTORE(Op, DAG); 138336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Ret.getNode()) { 138436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Ret; 138536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 1386c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Lowering for indirect addressing 1387c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1388c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const MachineFunction &MF = DAG.getMachineFunction(); 1389ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AMDGPUFrameLowering *TFL = 1390ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const AMDGPUFrameLowering *>(Subtarget->getFrameLowering()); 1391c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth = TFL->getStackWidth(MF); 1392c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1393c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1394c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1395c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ValueVT.isVector()) { 1396c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned NumElemVT = ValueVT.getVectorNumElements(); 1397c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ElemVT = ValueVT.getVectorElementType(); 1398dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 4> Stores(NumElemVT); 1399c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1400c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1401c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard "vector width in load"); 1402c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1403c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = 0; i < NumElemVT; ++i) { 1404c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned Channel, PtrIncr; 1405c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getStackAddress(StackWidth, i, Channel, PtrIncr); 1406c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 14076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(PtrIncr, DL, MVT::i32)); 1408c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SDValue Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, 14096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Value, DAG.getConstant(i, DL, MVT::i32)); 1410c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1411c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Stores[i] = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1412c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Elem, Ptr, 14136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(Channel, DL, MVT::i32)); 1414c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); 1416c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1417c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (ValueVT == MVT::i8) { 1418c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Value = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Value); 1419c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1420c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, Chain, Value, Ptr, 14216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32)); // Channel 1422c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1423c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1424c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return Chain; 1425f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1426f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 14279f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard// return (512 + (kc_bank << 12) 14289f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellardstatic int 14299f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom StellardConstantAddressBlock(unsigned AddressSpace) { 14309f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard switch (AddressSpace) { 14319f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_0: 14329f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512; 14339f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_1: 14349f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096; 14359f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_2: 14369f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 2; 14379f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_3: 14389f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 3; 14399f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_4: 14409f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 4; 14419f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_5: 14429f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 5; 14439f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_6: 14449f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 6; 14459f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_7: 14469f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 7; 14479f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_8: 14489f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 8; 14499f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_9: 14509f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 9; 14519f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_10: 14529f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 10; 14539f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_11: 14549f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 11; 14559f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_12: 14569f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 12; 14579f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_13: 14589f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 13; 14599f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_14: 14609f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 14; 14619f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case AMDGPUAS::CONSTANT_BUFFER_15: 14629f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return 512 + 4096 * 15; 14639f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard default: 14649f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return -1; 14659f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 14669f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard} 14679f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 14689f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom StellardSDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const 14699f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard{ 14709f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard EVT VT = Op.getValueType(); 1471ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 14729f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard LoadSDNode *LoadNode = cast<LoadSDNode>(Op); 14739f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Chain = Op.getOperand(0); 14749f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Ptr = Op.getOperand(1); 14759f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue LoweredLoad; 14769f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 1477cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar if (SDValue Ret = AMDGPUTargetLowering::LowerLOAD(Op, DAG)) 1478cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return Ret; 147936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 148037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Lower loads constant address space global variable loads 148137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (LoadNode->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS && 14824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar isa<GlobalVariable>(GetUnderlyingObject( 1483cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar LoadNode->getMemOperand()->getValue(), DAG.getDataLayout()))) { 148437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1485cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SDValue Ptr = DAG.getZExtOrTrunc( 1486cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar LoadNode->getBasePtr(), DL, 1487cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar getPointerTy(DAG.getDataLayout(), AMDGPUAS::PRIVATE_ADDRESS)); 148837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 14896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(2, DL, MVT::i32)); 149037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op->getVTList(), 149137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LoadNode->getChain(), Ptr, 14926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32), 14936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(2)); 149437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 149536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1496d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) { 1497d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard SDValue MergedValues[2] = { 149837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ScalarizeVectorLoad(Op, DAG), 1499d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard Chain 1500d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard }; 1501dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 1502d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard } 1503d08a9303614355cfdcac5f2c27c09ce809565423Tom Stellard 15049f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); 150529f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault if (ConstantBlock > -1 && 150629f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || 150729f1788de96cbf88ab87e3da130cf626b2e8e029Matt Arsenault (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { 15089f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Result; 1509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<ConstantExpr>(LoadNode->getMemOperand()->getValue()) || 1510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<Constant>(LoadNode->getMemOperand()->getValue()) || 15117a59defe70257302e2f68daba2fb8e1735e7cd0eMatt Arsenault isa<ConstantSDNode>(Ptr)) { 15129f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Slots[4]; 15139f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard for (unsigned i = 0; i < 4; i++) { 15149f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // We want Const position encoded with the following formula : 15159f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // (((512 + (kc_bank << 12) + const_index) << 2) + chan) 15169f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // const_index is Ptr computed by llvm using an alignment of 16. 15179f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and 15189f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // then div by 4 at the ISel step 15199f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 15206948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32)); 15219f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr); 15229f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 1523692ee102ebef535d311c35d53457028083e5c5beTom Stellard EVT NewVT = MVT::v4i32; 1524692ee102ebef535d311c35d53457028083e5c5beTom Stellard unsigned NumElements = 4; 1525692ee102ebef535d311c35d53457028083e5c5beTom Stellard if (VT.isVector()) { 1526692ee102ebef535d311c35d53457028083e5c5beTom Stellard NewVT = VT; 1527692ee102ebef535d311c35d53457028083e5c5beTom Stellard NumElements = VT.getVectorNumElements(); 1528692ee102ebef535d311c35d53457028083e5c5beTom Stellard } 1529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result = DAG.getNode(ISD::BUILD_VECTOR, DL, NewVT, 1530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines makeArrayRef(Slots, NumElements)); 15319f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } else { 153236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // non-constant ptr can't be folded, keeps it as a v4f32 load 15339f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32, 15346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, 15356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(4, DL, MVT::i32)), 15366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(LoadNode->getAddressSpace() - 15376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32) 15389f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard ); 15399f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15409f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 15419f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (!VT.isVector()) { 15429f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result, 15436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i32)); 15449f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15459f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 15469f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue MergedValues[2] = { 1547dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result, 1548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain 15499f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard }; 1550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 15519f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 15529f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard 1553dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // For most operations returning SDValue() will result in the node being 1554dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we 1555dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // need to manually expand loads that may be legal in some address spaces and 1556dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for 1557dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // compute shaders, since the data is sign extended when it is uploaded to the 1558dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // buffer. However SEXT loads from other address spaces are not supported, so 1559dbd936f6ccd2588e8e232d9843d45d3697dce1c3Matt Arsenault // we need to expand them here. 156058d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { 156158d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard EVT MemVT = LoadNode->getMemoryVT(); 156258d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); 156358d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard SDValue NewLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, Chain, Ptr, 156458d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->getPointerInfo(), MemVT, 156558d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->isVolatile(), 156658d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->isNonTemporal(), 156737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LoadNode->isInvariant(), 156858d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard LoadNode->getAlignment()); 15696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad, 15706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getValueType(MemVT)); 157158d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard 15726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue MergedValues[2] = { Res, Chain }; 1573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(MergedValues, DL); 157458d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard } 157558d3335cb9d2a40bd15c29a12ba045163295190eTom Stellard 1576c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { 1577c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard return SDValue(); 1578c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1579c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1580c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard // Lowering for indirect addressing 1581c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard const MachineFunction &MF = DAG.getMachineFunction(); 1582ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AMDGPUFrameLowering *TFL = 1583ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const AMDGPUFrameLowering *>(Subtarget->getFrameLowering()); 1584c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned StackWidth = TFL->getStackWidth(MF); 1585c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1586c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = stackPtrToRegIndex(Ptr, StackWidth, DAG); 1587c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1588c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard if (VT.isVector()) { 1589c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned NumElemVT = VT.getVectorNumElements(); 1590c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT ElemVT = VT.getVectorElementType(); 1591c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard SDValue Loads[4]; 1592c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1593c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard assert(NumElemVT >= StackWidth && "Stack width cannot be greater than " 1594c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard "vector width in load"); 1595c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1596c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = 0; i < NumElemVT; ++i) { 1597c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard unsigned Channel, PtrIncr; 1598c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard getStackAddress(StackWidth, i, Channel, PtrIncr); 1599c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr, 16006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(PtrIncr, DL, MVT::i32)); 1601c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Loads[i] = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, ElemVT, 1602c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Ptr, 16036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(Channel, DL, MVT::i32), 1604c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Op.getOperand(2)); 1605c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1606c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard for (unsigned i = NumElemVT; i < 4; ++i) { 1607c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Loads[i] = DAG.getUNDEF(ElemVT); 1608c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1609c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard EVT TargetVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 4); 1610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoweredLoad = DAG.getNode(ISD::BUILD_VECTOR, DL, TargetVT, Loads); 1611c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } else { 1612c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard LoweredLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, VT, 1613c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Chain, Ptr, 16146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32), // Channel 1615c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard Op.getOperand(2)); 1616c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard } 1617c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { 1619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoweredLoad, 1620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain 1621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 1622c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard 1623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 16249f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard} 1625f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1626c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { 1627c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Chain = Op.getOperand(0); 1628c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Cond = Op.getOperand(1); 1629c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Jump = Op.getOperand(2); 1630c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1631c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(), 1632c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Chain, Jump, Cond); 1633c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 1634c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1635f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// XXX Only kernel functions are supported, so we can assume for now that 1636f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// every function is a kernel function, but in the future we should use 1637f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// separate calling conventions for kernel and non-kernel functions. 1638f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::LowerFormalArguments( 1639f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain, 1640f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CallingConv::ID CallConv, 1641f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard bool isVarArg, 1642f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<ISD::InputArg> &Ins, 1643ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, SelectionDAG &DAG, 1644f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVectorImpl<SDValue> &InVals) const { 1645f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard SmallVector<CCValAssign, 16> ArgLocs; 164637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 164737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines *DAG.getContext()); 164870a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune MachineFunction &MF = DAG.getMachineFunction(); 164937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 1650f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 1651f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard SmallVector<ISD::InputArg, 8> LocalIns; 1652f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 1653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines getOriginalFunctionArgs(DAG, MF.getFunction(), Ins, LocalIns); 1654f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard 1655f95b1621887e3409ceec2db47e1b44271d934735Tom Stellard AnalyzeFormalArguments(CCInfo, LocalIns); 1656f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 1657a7eea0568c16f8e25b9e3ba9b7b73ae506738b63Tom Stellard for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 1658f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard CCValAssign &VA = ArgLocs[i]; 165937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const ISD::InputArg &In = Ins[i]; 166037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = In.VT; 166137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT MemVT = VA.getLocVT(); 166237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!VT.isVector() && MemVT.isVector()) { 166337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Get load source type if scalarized. 166437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MemVT = MemVT.getVectorElementType(); 166537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 16665864284d71ed89a4280e5171c389ad83fe183db7Tom Stellard 166737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (MFI->getShaderType() != ShaderType::COMPUTE) { 166870a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune unsigned Reg = MF.addLiveIn(VA.getLocReg(), &AMDGPU::R600_Reg128RegClass); 166970a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT); 167070a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune InVals.push_back(Register); 167170a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune continue; 167270a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune } 167370a7d5ddb4f00bbb61afe7b536c6f599f771ab9aVincent Lejeune 1674f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), 167537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AMDGPUAS::CONSTANT_BUFFER_0); 1676f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard 167736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // i64 isn't a legal type, so the register type used ends up as i32, which 167836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't expected here. It attempts to create this sextload, but it ends up 167936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // being invalid. Somehow this seems to work with i64 arguments, but breaks 168036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for <1 x i64>. 168136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1682f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard // The first 36 bytes of the input buffer contains information about 1683f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard // thread group and global sizes. 168437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ISD::LoadExtType Ext = ISD::NON_EXTLOAD; 168537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) { 168637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: This should really check the extload type, but the handling of 168737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // extload vector parameters seems to be broken. 168837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 168937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD; 169037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Ext = ISD::SEXTLOAD; 169137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 169237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 169337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Compute the offset from the value. 169437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // XXX - I think PartOffset should give you this, but it seems to give the 169537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // size of the register which isn't useful. 169637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1697ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned ValBase = ArgLocs[In.getOrigArgIndex()].getLocMemOffset(); 169837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned PartOffset = VA.getLocMemOffset(); 169937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Offset = 36 + VA.getLocMemOffset(); 1700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 170137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MachinePointerInfo PtrInfo(UndefValue::get(PtrTy), PartOffset - ValBase); 170237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Arg = DAG.getLoad(ISD::UNINDEXED, Ext, VT, DL, Chain, 17036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Offset, DL, MVT::i32), 170437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getUNDEF(MVT::i32), 170537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PtrInfo, 170637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MemVT, false, true, true, 4); 1707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 4 is the preferred alignment for the CONSTANT memory space. 1709f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard InVals.push_back(Arg); 171037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MFI->ABIArgOffset = Offset + MemVT.getStoreSize(); 1711f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1712f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Chain; 1713f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1714f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1715cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga NainarEVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 1716cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar EVT VT) const { 1717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 1718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MVT::i32; 1719f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return VT.changeVectorElementTypeToInteger(); 1720f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 1721f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue CompactSwizzlableVector( 1723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG, SDValue VectorEntry, 1724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<unsigned, unsigned> &RemapSwizzle) { 172598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 172698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(RemapSwizzle.empty()); 172798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewBldVec[4] = { 1728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(0), 1729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(1), 1730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(2), 1731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getOperand(3) 173298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 173398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 173498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 1735f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::UNDEF) 1736f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // We mask write here to teach later passes that the ith element of this 1737f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // vector is undef. Thus we can use it to reduce 128 bits reg usage, 1738f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune // break false dependencies and additionnaly make assembly easier to read. 1739f2b3a569ae25dbba264cef93602b4147d2a723d6Vincent Lejeune RemapSwizzle[i] = 7; // SEL_MASK_WRITE 174098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) { 174198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (C->isZero()) { 174298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = 4; // SEL_0 174398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(MVT::f32); 174498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } else if (C->isExactlyValue(1.0)) { 174598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = 5; // SEL_1 174698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(MVT::f32); 174798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 174898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 174998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 175098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::UNDEF) 175198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune continue; 175298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned j = 0; j < i; j++) { 175398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i] == NewBldVec[j]) { 175498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); 175598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune RemapSwizzle[i] = j; 175698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune break; 175798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 175898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 175998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 176098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 176198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(VectorEntry), 1762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getValueType(), NewBldVec); 176398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 176498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 17656c59c7a6fdc5b58de4f3c349b59a567c239818e4Benjamin Kramerstatic SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, 17666c59c7a6fdc5b58de4f3c349b59a567c239818e4Benjamin Kramer DenseMap<unsigned, unsigned> &RemapSwizzle) { 176798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(VectorEntry.getOpcode() == ISD::BUILD_VECTOR); 176898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(RemapSwizzle.empty()); 176998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewBldVec[4] = { 177098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(0), 177198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(1), 177298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(2), 177398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune VectorEntry.getOperand(3) 177498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 177598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune bool isUnmovable[4] = { false, false, false, false }; 177636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0; i < 4; i++) { 1777f4bdec2ebeb1306a77e9377583c5799199775f88Vincent Lejeune RemapSwizzle[i] = i; 177836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 177936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 178036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ->getZExtValue(); 178136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (i == Idx) 178236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines isUnmovable[Idx] = true; 178336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 178598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 178698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 178798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 178898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune unsigned Idx = dyn_cast<ConstantSDNode>(NewBldVec[i].getOperand(1)) 178998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune ->getZExtValue(); 179091ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune if (isUnmovable[Idx]) 179191ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune continue; 179291ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune // Swap i and Idx 179391ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune std::swap(NewBldVec[Idx], NewBldVec[i]); 179491ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune std::swap(RemapSwizzle[i], RemapSwizzle[Idx]); 179591ec4b0cac7a7476a9d30d6f1adbf218ee6673a0Vincent Lejeune break; 179698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 179798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 179898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 179998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(VectorEntry), 1800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorEntry.getValueType(), NewBldVec); 180198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 180298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 180398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 180498017a015bb862afce4f90d432eded4e28fe1d26Vincent LejeuneSDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, 18056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Swz[4], SelectionDAG &DAG, 18066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL) const { 180798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune assert(BuildVector.getOpcode() == ISD::BUILD_VECTOR); 180898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune // Old -> New swizzle values 180998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune DenseMap<unsigned, unsigned> SwizzleRemap; 181098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 181198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap); 181298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 18130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 181498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 18156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 181698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 181798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 181898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SwizzleRemap.clear(); 181998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap); 182098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune for (unsigned i = 0; i < 4; i++) { 18210c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue(); 182298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (SwizzleRemap.find(Idx) != SwizzleRemap.end()) 18236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32); 182498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 182598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 182698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune return BuildVector; 182798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune} 182898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 182998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 1830f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 1831f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Custom DAG Optimizations 1832f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 1833f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1834f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue R600TargetLowering::PerformDAGCombine(SDNode *N, 1835f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAGCombinerInfo &DCI) const { 1836f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG = DCI.DAG; 1837f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1838f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (N->getOpcode()) { 1839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1840f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) 1841f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::FP_ROUND: { 1842f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Arg = N->getOperand(0); 1843f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) { 1844ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), N->getValueType(0), 1845f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Arg.getOperand(0)); 1846f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 1847f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 1848f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 18491234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 18501234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> 18511234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // (i32 select_cc f32, f32, -1, 0 cc) 18521234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // 18531234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // Mesa's GLSL frontend generates the above pattern a lot and we can lower 18541234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // this to one of the SET*_DX10 instructions. 18551234c9be42b4ebd4b398df461123205dccf3706cTom Stellard case ISD::FP_TO_SINT: { 18561234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue FNeg = N->getOperand(0); 18571234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (FNeg.getOpcode() != ISD::FNEG) { 18581234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 18591234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 18601234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue SelectCC = FNeg.getOperand(0); 18611234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (SelectCC.getOpcode() != ISD::SELECT_CC || 18621234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS 18631234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(2).getValueType() != MVT::f32 || // True 18641234c9be42b4ebd4b398df461123205dccf3706cTom Stellard !isHWTrueValue(SelectCC.getOperand(2)) || 18651234c9be42b4ebd4b398df461123205dccf3706cTom Stellard !isHWFalseValue(SelectCC.getOperand(3))) { 18661234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 18671234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 18681234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 18696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 18706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SELECT_CC, dl, N->getValueType(0), 18711234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(0), // LHS 18721234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(1), // RHS 18736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-1, dl, MVT::i32), // True 18746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32), // False 18751234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SelectCC.getOperand(4)); // CC 18761234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 18771234c9be42b4ebd4b398df461123205dccf3706cTom Stellard break; 18781234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 187915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 18801fe9069d53f586963d61523f7c5a7d41d80a9d8bNAKAMURA Takumi // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx 18811fe9069d53f586963d61523f7c5a7d41d80a9d8bNAKAMURA Takumi // => build_vector elt0, ... , NewEltIdx, ... , eltN 188215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet case ISD::INSERT_VECTOR_ELT: { 188315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue InVec = N->getOperand(0); 188415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue InVal = N->getOperand(1); 188515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDValue EltNo = N->getOperand(2); 188615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SDLoc dl(N); 188715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 188815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // If the inserted element is an UNDEF, just use the input vector. 188915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVal.getOpcode() == ISD::UNDEF) 189015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return InVec; 189115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 189215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet EVT VT = InVec.getValueType(); 189315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 189415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // If we can't generate a legal BUILD_VECTOR, exit 189515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (!isOperationLegal(ISD::BUILD_VECTOR, VT)) 189615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 189715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 189815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Check that we know which element is being inserted 189915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (!isa<ConstantSDNode>(EltNo)) 190015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 190115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 190215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 190315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 190415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 190515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // vector elements. 190615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet SmallVector<SDValue, 8> Ops; 190715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVec.getOpcode() == ISD::BUILD_VECTOR) { 190815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops.append(InVec.getNode()->op_begin(), 190915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet InVec.getNode()->op_end()); 191015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } else if (InVec.getOpcode() == ISD::UNDEF) { 191115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet unsigned NElts = VT.getVectorNumElements(); 191215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 191315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } else { 191415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet return SDValue(); 191515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 191615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 191715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Insert the element 191815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (Elt < Ops.size()) { 191915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // All the operands of BUILD_VECTOR must have the same type; 192015d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // we enforce that here. 192115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet EVT OpVT = Ops[0].getValueType(); 192215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet if (InVal.getValueType() != OpVT) 192315d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet InVal = OpVT.bitsGT(InVal.getValueType()) ? 192415d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : 192515d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); 192615d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet Ops[Elt] = InVal; 192715d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 192815d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 192915d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet // Return the new vector 1930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 193115d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet } 193215d1b85094cf4c1520fdfd12db2111cd36a194dbQuentin Colombet 19339f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // Extract_vec (Build_vector) generated by custom lowering 19349f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard // also needs to be customly combined 19359f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard case ISD::EXTRACT_VECTOR_ELT: { 19369f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard SDValue Arg = N->getOperand(0); 19379f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (Arg.getOpcode() == ISD::BUILD_VECTOR) { 19389f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 19399f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard unsigned Element = Const->getZExtValue(); 19409f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard return Arg->getOperand(Element); 19419f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 19429f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 1943cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard if (Arg.getOpcode() == ISD::BITCAST && 1944cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { 1945cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 1946cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard unsigned Element = Const->getZExtValue(); 1947ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getVTList(), 1948cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard Arg->getOperand(0).getOperand(Element)); 1949cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard } 1950cacbcb0f2c60d45618dee0e10ded2ed2052166a6Tom Stellard } 1951cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar break; 19529f7818d9bdfce2e9c7a2cbe31490a135aa6d1211Tom Stellard } 19531234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19541234c9be42b4ebd4b398df461123205dccf3706cTom Stellard case ISD::SELECT_CC: { 1955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Try common optimizations 1956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 1957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Ret.getNode()) 1958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Ret; 1959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 19601234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> 19611234c9be42b4ebd4b398df461123205dccf3706cTom Stellard // selectcc x, y, a, b, inv(cc) 19627893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // 19637893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> 19647893d29c62146baddf43c4d9d42678d246a52feaTom Stellard // selectcc x, y, a, b, cc 19651234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue LHS = N->getOperand(0); 19661234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (LHS.getOpcode() != ISD::SELECT_CC) { 19671234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 19681234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 19691234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19701234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue RHS = N->getOperand(1); 19711234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue True = N->getOperand(2); 19721234c9be42b4ebd4b398df461123205dccf3706cTom Stellard SDValue False = N->getOperand(3); 19737893d29c62146baddf43c4d9d42678d246a52feaTom Stellard ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 19741234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19751234c9be42b4ebd4b398df461123205dccf3706cTom Stellard if (LHS.getOperand(2).getNode() != True.getNode() || 19761234c9be42b4ebd4b398df461123205dccf3706cTom Stellard LHS.getOperand(3).getNode() != False.getNode() || 19777893d29c62146baddf43c4d9d42678d246a52feaTom Stellard RHS.getNode() != False.getNode()) { 19781234c9be42b4ebd4b398df461123205dccf3706cTom Stellard return SDValue(); 19791234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 19801234c9be42b4ebd4b398df461123205dccf3706cTom Stellard 19817893d29c62146baddf43c4d9d42678d246a52feaTom Stellard switch (NCC) { 19827893d29c62146baddf43c4d9d42678d246a52feaTom Stellard default: return SDValue(); 19837893d29c62146baddf43c4d9d42678d246a52feaTom Stellard case ISD::SETNE: return LHS; 19847893d29c62146baddf43c4d9d42678d246a52feaTom Stellard case ISD::SETEQ: { 19857893d29c62146baddf43c4d9d42678d246a52feaTom Stellard ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); 19867893d29c62146baddf43c4d9d42678d246a52feaTom Stellard LHSCC = ISD::getSetCCInverse(LHSCC, 19877893d29c62146baddf43c4d9d42678d246a52feaTom Stellard LHS.getOperand(0).getValueType().isInteger()); 198812d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard if (DCI.isBeforeLegalizeOps() || 198912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) 199012d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard return DAG.getSelectCC(SDLoc(N), 199112d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(0), 199212d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(1), 199312d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(2), 199412d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHS.getOperand(3), 199512d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard LHSCC); 199612d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard break; 1997abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune } 19987893d29c62146baddf43c4d9d42678d246a52feaTom Stellard } 199912d43f9baf83b6a2cc444c89bb688ebfe01a9fa1Tom Stellard return SDValue(); 20007893d29c62146baddf43c4d9d42678d246a52feaTom Stellard } 20014c52d450dc3968267d1f089d36397fc785dcc7b4Tom Stellard 2002abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune case AMDGPUISD::EXPORT: { 2003abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue Arg = N->getOperand(1); 2004abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune if (Arg.getOpcode() != ISD::BUILD_VECTOR) 2005abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune break; 200698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 2007abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue NewArgs[8] = { 2008abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(0), // Chain 2009abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune SDValue(), 2010abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(2), // ArrayBase 2011abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(3), // Type 2012abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(4), // SWZ_X 2013abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(5), // SWZ_Y 2014abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(6), // SWZ_Z 2015abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune N->getOperand(7) // SWZ_W 2016abfd5f6154b10cc5801bc9e1b8e8221df0113c68Vincent Lejeune }; 2017ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(N); 20186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL); 2019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AMDGPUISD::EXPORT, DL, N->getVTList(), NewArgs); 20201234c9be42b4ebd4b398df461123205dccf3706cTom Stellard } 202198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune case AMDGPUISD::TEXTURE_FETCH: { 202298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue Arg = N->getOperand(1); 202398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune if (Arg.getOpcode() != ISD::BUILD_VECTOR) 202498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune break; 202598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune 202698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune SDValue NewArgs[19] = { 202798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(0), 202898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(1), 202998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(2), 203098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(3), 203198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(4), 203298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(5), 203398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(6), 203498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(7), 203598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(8), 203698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(9), 203798017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(10), 203898017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(11), 203998017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(12), 204098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(13), 204198017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(14), 204298017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(15), 204398017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(16), 204498017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(17), 204598017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune N->getOperand(18), 204698017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune }; 20476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(N); 20486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL); 20496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs); 205098017a015bb862afce4f90d432eded4e28fe1d26Vincent Lejeune } 2051f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 2052dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AMDGPUTargetLowering::PerformDAGCombine(N, DCI); 2054f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 2055fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2056fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeunestatic bool 2057fe7831861432d71de47ce502e799fb7264b9f24cVincent LejeuneFoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg, 2058f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Abs, SDValue &Sel, SDValue &Imm, SelectionDAG &DAG) { 2059fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune const R600InstrInfo *TII = 206037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static_cast<const R600InstrInfo *>(DAG.getSubtarget().getInstrInfo()); 2061fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Src.isMachineOpcode()) 2062fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2063fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune switch (Src.getMachineOpcode()) { 2064fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::FNEG_R600: 2065fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Neg.getNode()) 2066fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2067fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = Src.getOperand(0); 20686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2069fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2070fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::FABS_R600: 2071fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Abs.getNode()) 2072fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2073fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = Src.getOperand(0); 20746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32); 2075fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2076fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune case AMDGPU::CONST_COPY: { 2077fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune unsigned Opcode = ParentNode->getMachineOpcode(); 2078fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2079fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2080fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Sel.getNode()) 2081fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2082fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2083fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue CstOffset = Src.getOperand(0); 2084fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (ParentNode->getValueType(0).isVector()) 2085fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2086fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2087fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune // Gather constants values 2088fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SrcIndices[] = { 2089fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2090fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2091fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2), 2092fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2093fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2094fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2095fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2096fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2097fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2098fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2099fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2100fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2101fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune std::vector<unsigned> Consts; 2102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int OtherSrcIdx : SrcIndices) { 2103fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); 2104fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OtherSrcIdx < 0 || OtherSelIdx < 0) 2105fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune continue; 2106fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (HasDst) { 2107fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune OtherSrcIdx--; 2108fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune OtherSelIdx--; 2109fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2110fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (RegisterSDNode *Reg = 2111fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { 2112fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (Reg->getReg() == AMDGPU::ALU_CONST) { 2113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Cst 2114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx)); 2115fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Consts.push_back(Cst->getZExtValue()); 2116fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2117fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2118fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2119fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset); 2121fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Consts.push_back(Cst->getZExtValue()); 2122fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!TII->fitsConstReadLimitations(Consts)) { 2123fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2124fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2125fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2126fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Sel = CstOffset; 2127fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune Src = DAG.getRegister(AMDGPU::ALU_CONST, MVT::f32); 2128fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return true; 2129fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2130f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune case AMDGPU::MOV_IMM_I32: 2131f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune case AMDGPU::MOV_IMM_F32: { 2132f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune unsigned ImmReg = AMDGPU::ALU_LITERAL_X; 2133f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune uint64_t ImmValue = 0; 2134f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2135f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2136f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (Src.getMachineOpcode() == AMDGPU::MOV_IMM_F32) { 2137f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantFPSDNode *FPC = dyn_cast<ConstantFPSDNode>(Src.getOperand(0)); 2138f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune float FloatValue = FPC->getValueAPF().convertToFloat(); 2139f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FloatValue == 0.0) { 2140f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ZERO; 2141f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (FloatValue == 0.5) { 2142f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::HALF; 2143f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (FloatValue == 1.0) { 2144f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ONE; 2145f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2146f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); 2147f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2148f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2149f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src.getOperand(0)); 2150f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune uint64_t Value = C->getZExtValue(); 2151f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (Value == 0) { 2152f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ZERO; 2153f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (Value == 1) { 2154f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmReg = AMDGPU::ONE_INT; 2155f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else { 2156f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmValue = Value; 2157f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2158f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2159f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune 2160f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // Check that we aren't already using an immediate. 2161f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // XXX: It's possible for an instruction to have more than one 2162f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune // immediate operand, but this is not supported yet. 2163f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (ImmReg == AMDGPU::ALU_LITERAL_X) { 2164f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (!Imm.getNode()) 2165f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return false; 2166f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ConstantSDNode *C = dyn_cast<ConstantSDNode>(Imm); 2167f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune assert(C); 2168f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (C->getZExtValue()) 2169f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return false; 21706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32); 2171f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2172f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune Src = DAG.getRegister(ImmReg, MVT::i32); 2173f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return true; 2174f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2175fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune default: 2176fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return false; 2177fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2178fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune} 2179fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2180fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2181fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune/// \brief Fold the instructions after selecting them 2182fe7831861432d71de47ce502e799fb7264b9f24cVincent LejeuneSDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node, 2183fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelectionDAG &DAG) const { 2184fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune const R600InstrInfo *TII = 218537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static_cast<const R600InstrInfo *>(DAG.getSubtarget().getInstrInfo()); 2186fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!Node->isMachineOpcode()) 2187fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2188fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune unsigned Opcode = Node->getMachineOpcode(); 2189fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue FakeOp; 2190fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2191ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines std::vector<SDValue> Ops(Node->op_begin(), Node->op_end()); 2192fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2193fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (Opcode == AMDGPU::DOT_4) { 2194fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OperandIdx[] = { 2195fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X), 2196fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y), 2197fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z), 2198fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W), 2199fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X), 2200fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y), 2201fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z), 2202fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W) 2203661bd3df7518a3d984dada66473602a0401618baNAKAMURA Takumi }; 2204fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int NegIdx[] = { 2205fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_X), 2206fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Y), 2207fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_Z), 2208fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg_W), 2209fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_X), 2210fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Y), 2211fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_Z), 2212fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg_W) 2213fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2214fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int AbsIdx[] = { 2215fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_X), 2216fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Y), 2217fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_Z), 2218fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs_W), 2219fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_X), 2220fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Y), 2221fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_Z), 2222fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs_W) 2223fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2224fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune for (unsigned i = 0; i < 8; i++) { 2225fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OperandIdx[i] < 0) 2226fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2227fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Src = Ops[OperandIdx[i] - 1]; 2228fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Neg = Ops[NegIdx[i] - 1]; 2229fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Abs = Ops[AbsIdx[i] - 1]; 2230fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2231fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2232fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (HasDst) 2233fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelIdx--; 2234fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2235f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG)) 2236f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2237f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2238f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } else if (Opcode == AMDGPU::REG_SEQUENCE) { 2239f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { 2240f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Src = Ops[i]; 2241f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG)) 2242fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2243fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 22445251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune } else if (Opcode == AMDGPU::CLAMP_R600) { 22455251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune SDValue Src = Node->getOperand(0); 22465251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune if (!Src.isMachineOpcode() || 22475251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune !TII->hasInstrModifiers(Src.getMachineOpcode())) 22485251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune return Node; 22495251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune int ClampIdx = TII->getOperandIdx(Src.getMachineOpcode(), 22505251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune AMDGPU::OpName::clamp); 22515251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune if (ClampIdx < 0) 22525251d180f4af15414a9c7ae5723dd48bc938576bVincent Lejeune return Node; 22536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Node); 2254ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines std::vector<SDValue> Ops(Src->op_begin(), Src->op_end()); 22556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops[ClampIdx - 1] = DAG.getTargetConstant(1, DL, MVT::i32); 22566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getMachineNode(Src.getMachineOpcode(), DL, 22576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Node->getVTList(), Ops); 2258fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } else { 2259fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (!TII->hasInstrModifiers(Opcode)) 2260fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2261fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int OperandIdx[] = { 2262fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0), 2263fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1), 2264fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2) 2265fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2266fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int NegIdx[] = { 2267fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_neg), 2268fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_neg), 2269fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src2_neg) 2270fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2271fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int AbsIdx[] = { 2272fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_abs), 2273fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_abs), 2274fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune -1 2275fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune }; 2276fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune for (unsigned i = 0; i < 3; i++) { 2277fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune if (OperandIdx[i] < 0) 2278fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2279fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Src = Ops[OperandIdx[i] - 1]; 2280fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Neg = Ops[NegIdx[i] - 1]; 2281fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue FakeAbs; 2282fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; 2283fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1; 2284fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); 2285f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune int ImmIdx = TII->getOperandIdx(Opcode, AMDGPU::OpName::literal); 2286f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (HasDst) { 2287fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SelIdx--; 2288f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune ImmIdx--; 2289f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune } 2290fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; 2291f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune SDValue &Imm = Ops[ImmIdx]; 2292f57d692c11f0ff6e9c45d2c48c5f362f4c575cf7Vincent Lejeune if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG)) 2293fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); 2294fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2295fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune } 2296fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune 2297fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune return Node; 2298fe7831861432d71de47ce502e799fb7264b9f24cVincent Lejeune} 2299