169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// 313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// The LLVM Compiler Infrastructure 413c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// 813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 913c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// 1069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// This file performs vector type splitting and scalarization for LegalizeTypes. 1169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// Scalarization is the act of changing a computation in an illegal one-element 1269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// vector type to be a computation in its scalar element type. For example, 1369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 1469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// as a base case when scalarizing vector arithmetic like <4 x f32>, which 1569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 1669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// types. 1769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// Splitting is the act of changing a computation in an invalid vector type to 18f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands// be a computation in two vectors of half the size. For example, implementing 19f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands// <128 x f32> operations in terms of two <64 x f32> operations. 2013c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// 2113c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 2213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner 2313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner#include "LegalizeTypes.h" 240b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 257d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin#include "llvm/Support/ErrorHandling.h" 264437ae213d5435390f0750213b53ec807c047f22Chris Lattner#include "llvm/Support/raw_ostream.h" 2713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattnerusing namespace llvm; 2813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner 2969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands//===----------------------------------------------------------------------===// 3069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// Result Vector Scalarization: <1 x ty> -> ty. 3169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands//===----------------------------------------------------------------------===// 3269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 33f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sandsvoid DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 347419b1806a1c3abdc23b62de76fae737b763fb33David Greene DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 354437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 367419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 37475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue R = SDValue(); 3869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 3969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands switch (N->getOpcode()) { 4069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands default: 4169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands#ifndef NDEBUG 427419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 434437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 447419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 4569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands#endif 465aa5d574f464ff9ff15a4c01360aaabc9bdc8a8fNick Lewycky report_fatal_error("Do not know how to scalarize the result of this " 475aa5d574f464ff9ff15a4c01360aaabc9bdc8a8fNick Lewycky "operator!\n"); 4869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 494c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 50bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 5146646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 52331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 53d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 54b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 55d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 56331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 5705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 5805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 59331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 60d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 612ce63c73520cd6e715f9114589f802938b5db01fPete Cooper case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 62331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 63331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 648c899ee031481dbece5f111379a274c848cb5902Duncan Sands case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 65331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 66331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 67e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::ANY_EXTEND: 684ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::CTLZ: 694ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::CTPOP: 704ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::CTTZ: 714ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FABS: 72e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FCEIL: 734ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FCOS: 74e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FEXP: 75e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FEXP2: 76e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FFLOOR: 77e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG: 78e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG10: 79e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG2: 80e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FNEARBYINT: 814ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FNEG: 82e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FP_EXTEND: 834ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FP_TO_SINT: 844ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FP_TO_UINT: 85e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FRINT: 864ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FSIN: 874ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FSQRT: 88509e84fa7146175c86dec5ef2167290f294dc89eDan Gohman case ISD::FTRUNC: 89e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::SIGN_EXTEND: 904ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::SINT_TO_FP: 91d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::TRUNCATE: 92e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::UINT_TO_FP: 9364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ZERO_EXTEND: 9464a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner R = ScalarizeVecRes_UnaryOp(N); 9564a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 964ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands 9769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::ADD: 984ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::AND: 9969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::FADD: 1004ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FDIV: 1014ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FMUL: 1024ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FPOW: 1034ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::FREM: 10469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::FSUB: 10569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::MUL: 1064ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::OR: 10769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::SDIV: 10869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::SREM: 1094ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::SUB: 1104ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands case ISD::UDIV: 11169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::UREM: 11264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::XOR: 11393b3b928d70df956b81ab1bffb7e8bab4b0c8fd7Mon P Wang case ISD::SHL: 11493b3b928d70df956b81ab1bffb7e8bab4b0c8fd7Mon P Wang case ISD::SRA: 11564a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::SRL: 11687c411b5cc254a8d169b834f3487657a8dc1e17aChris Lattner R = ScalarizeVecRes_BinOp(N); 11764a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 118b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper case ISD::FMA: 119b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper R = ScalarizeVecRes_TernaryOp(N); 120b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper break; 12169b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands } 12269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 12369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands // If R is null, the sub-method took care of registering the result. 124ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (R.getNode()) 125475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SetScalarizedVector(SDValue(N, ResNo), R); 12669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 12769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 128475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 129475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue LHS = GetScalarizedVector(N->getOperand(0)); 130475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue RHS = GetScalarizedVector(N->getOperand(1)); 131ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), 132644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen LHS.getValueType(), LHS, RHS); 13369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 13469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 135b49998d76cb4e414d13d60116adf13b085d85dc1Pete CooperSDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 136b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 137b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 138b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 139ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), 140b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper Op0.getValueType(), Op0, Op1, Op2); 141b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper} 142b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper 1434c19e12d28749c717d3b384962c9ec92796af1c9Duncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 1444c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands unsigned ResNo) { 1454c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 14662bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman return GetScalarizedVector(Op); 14762bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman} 14862bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman 149bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley PeckSDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 150e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT NewVT = N->getValueType(0).getVectorElementType(); 151ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BITCAST, SDLoc(N), 152644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen NewVT, N->getOperand(0)); 15369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 15469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 15546646572f76513e39bcdd0e06c66668ec1caf5bcNadav RotemSDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 15646646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem EVT EltVT = N->getValueType(0).getVectorElementType(); 15746646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem SDValue InOp = N->getOperand(0); 158c76fa8937d483ae83f94d0793254dbab78877687Nadav Rotem // The BUILD_VECTOR operands may be of wider element types and 159c76fa8937d483ae83f94d0793254dbab78877687Nadav Rotem // we may need to truncate them back to the requested return type. 160c76fa8937d483ae83f94d0793254dbab78877687Nadav Rotem if (EltVT.isInteger()) 161ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 16246646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem return InOp; 16346646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem} 16446646572f76513e39bcdd0e06c66668ec1caf5bcNadav Rotem 16577cdf30742284a173fe818417eb482224cdee8d4Mon P WangSDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 166e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT NewVT = N->getValueType(0).getVectorElementType(); 16777cdf30742284a173fe818417eb482224cdee8d4Mon P Wang SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 168ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getConvertRndSat(NewVT, SDLoc(N), 169c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen Op0, DAG.getValueType(NewVT), 17077cdf30742284a173fe818417eb482224cdee8d4Mon P Wang DAG.getValueType(Op0.getValueType()), 17177cdf30742284a173fe818417eb482224cdee8d4Mon P Wang N->getOperand(3), 17277cdf30742284a173fe818417eb482224cdee8d4Mon P Wang N->getOperand(4), 17377cdf30742284a173fe818417eb482224cdee8d4Mon P Wang cast<CvtRndSatSDNode>(N)->getCvtCode()); 17477cdf30742284a173fe818417eb482224cdee8d4Mon P Wang} 17577cdf30742284a173fe818417eb482224cdee8d4Mon P Wang 176d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 177ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 178d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands N->getValueType(0).getVectorElementType(), 179d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands N->getOperand(0), N->getOperand(1)); 180d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands} 181d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 182b141099c14bfa86167b2137e8a9544c6ee805955Eli FriedmanSDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 183b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman EVT NewVT = N->getValueType(0).getVectorElementType(); 184b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue Op = GetScalarizedVector(N->getOperand(0)); 185ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 186b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman NewVT, Op, N->getOperand(1)); 187b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman} 188b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman 189475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 190475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Op = GetScalarizedVector(N->getOperand(0)); 191ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::FPOWI, SDLoc(N), 192644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Op.getValueType(), Op, N->getOperand(1)); 19369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 19469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 195475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 19669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands // The value to insert may have a wider type than the vector element type, 19769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands // so be sure to truncate it to the element type if necessary. 198475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Op = N->getOperand(1); 199e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = N->getValueType(0).getVectorElementType(); 20028c05ac995c8063ecf4e05ef7c0879467ca98221Duncan Sands if (Op.getValueType() != EltVT) 20128c05ac995c8063ecf4e05ef7c0879467ca98221Duncan Sands // FIXME: Can this happen for floating point types? 202ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 20369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands return Op; 20469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 20569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 206475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 2077f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman assert(N->isUnindexed() && "Indexed vector load?"); 2087f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 209bcc8017c738e92d9c1af221b11c4916cb524184eEvan Cheng SDValue Result = DAG.getLoad(ISD::UNINDEXED, 210644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen N->getExtensionType(), 2117f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman N->getValueType(0).getVectorElementType(), 212ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc(N), 2137f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman N->getChain(), N->getBasePtr(), 214e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen DAG.getUNDEF(N->getBasePtr().getValueType()), 2155cd9391a089d146d8a18ade602769502e5a8d36fChris Lattner N->getPointerInfo(), 2167f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman N->getMemoryVT().getVectorElementType(), 2171e559443a17d1b335f697551c6263ba60d5dd827David Greene N->isVolatile(), N->isNonTemporal(), 218d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper N->isInvariant(), N->getOriginalAlignment()); 21905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 22005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // Legalized the chain result - switch anything that used the old chain to 22105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // use the new one. 222475871a144eb604ddaf37503397ba0941442e5fbDan Gohman ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 22305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands return Result; 22469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 22569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 226475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 2274ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 228e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT DestVT = N->getValueType(0).getVectorElementType(); 229475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Op = GetScalarizedVector(N->getOperand(0)); 230ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 23105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 23205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 233d1996360399ad6dbe75ee185b661b16c83146373Dan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 234d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman EVT EltVT = N->getValueType(0).getVectorElementType(); 235d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 236d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 238d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman LHS, DAG.getValueType(ExtVT)); 239d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman} 240d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman 241331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 242b10b5ac8d9da43ca2db61401a20af6b676c98438Duncan Sands // If the operand is wider than the vector element type then it is implicitly 243b10b5ac8d9da43ca2db61401a20af6b676c98438Duncan Sands // truncated. Make that explicit here. 244e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = N->getValueType(0).getVectorElementType(); 245b10b5ac8d9da43ca2db61401a20af6b676c98438Duncan Sands SDValue InOp = N->getOperand(0); 246b10b5ac8d9da43ca2db61401a20af6b676c98438Duncan Sands if (InOp.getValueType() != EltVT) 247ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 248b10b5ac8d9da43ca2db61401a20af6b676c98438Duncan Sands return InOp; 24969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 25069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 2512ce63c73520cd6e715f9114589f802938b5db01fPete CooperSDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 2522ce63c73520cd6e715f9114589f802938b5db01fPete Cooper SDValue Cond = GetScalarizedVector(N->getOperand(0)); 2532ce63c73520cd6e715f9114589f802938b5db01fPete Cooper SDValue LHS = GetScalarizedVector(N->getOperand(1)); 2542ce63c73520cd6e715f9114589f802938b5db01fPete Cooper TargetLowering::BooleanContent ScalarBool = TLI.getBooleanContents(false); 2552ce63c73520cd6e715f9114589f802938b5db01fPete Cooper TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true); 2562ce63c73520cd6e715f9114589f802938b5db01fPete Cooper if (ScalarBool != VecBool) { 2572ce63c73520cd6e715f9114589f802938b5db01fPete Cooper EVT CondVT = Cond.getValueType(); 2582ce63c73520cd6e715f9114589f802938b5db01fPete Cooper switch (ScalarBool) { 2592ce63c73520cd6e715f9114589f802938b5db01fPete Cooper case TargetLowering::UndefinedBooleanContent: 2602ce63c73520cd6e715f9114589f802938b5db01fPete Cooper break; 2612ce63c73520cd6e715f9114589f802938b5db01fPete Cooper case TargetLowering::ZeroOrOneBooleanContent: 2622ce63c73520cd6e715f9114589f802938b5db01fPete Cooper assert(VecBool == TargetLowering::UndefinedBooleanContent || 2632ce63c73520cd6e715f9114589f802938b5db01fPete Cooper VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 2642ce63c73520cd6e715f9114589f802938b5db01fPete Cooper // Vector read from all ones, scalar expects a single 1 so mask. 265ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 2662ce63c73520cd6e715f9114589f802938b5db01fPete Cooper Cond, DAG.getConstant(1, CondVT)); 2672ce63c73520cd6e715f9114589f802938b5db01fPete Cooper break; 2682ce63c73520cd6e715f9114589f802938b5db01fPete Cooper case TargetLowering::ZeroOrNegativeOneBooleanContent: 2692ce63c73520cd6e715f9114589f802938b5db01fPete Cooper assert(VecBool == TargetLowering::UndefinedBooleanContent || 2702ce63c73520cd6e715f9114589f802938b5db01fPete Cooper VecBool == TargetLowering::ZeroOrOneBooleanContent); 2712ce63c73520cd6e715f9114589f802938b5db01fPete Cooper // Vector reads from a one, scalar from all ones so sign extend. 272ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 2732ce63c73520cd6e715f9114589f802938b5db01fPete Cooper Cond, DAG.getValueType(MVT::i1)); 2742ce63c73520cd6e715f9114589f802938b5db01fPete Cooper break; 2752ce63c73520cd6e715f9114589f802938b5db01fPete Cooper } 2762ce63c73520cd6e715f9114589f802938b5db01fPete Cooper } 277b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault 278b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault return DAG.getSelect(SDLoc(N), 279b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault LHS.getValueType(), Cond, LHS, 280b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault GetScalarizedVector(N->getOperand(2))); 2812ce63c73520cd6e715f9114589f802938b5db01fPete Cooper} 2822ce63c73520cd6e715f9114589f802938b5db01fPete Cooper 283475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 284475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue LHS = GetScalarizedVector(N->getOperand(1)); 285b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault return DAG.getSelect(SDLoc(N), 286b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault LHS.getValueType(), N->getOperand(0), LHS, 287b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault GetScalarizedVector(N->getOperand(2))); 28869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 28969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 290e59416efb8472ee95c4500a94d2048c585faff6dDuncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 291e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands SDValue LHS = GetScalarizedVector(N->getOperand(2)); 292ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 293e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands N->getOperand(0), N->getOperand(1), 294e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands LHS, GetScalarizedVector(N->getOperand(3)), 295e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands N->getOperand(4)); 296e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands} 297e59416efb8472ee95c4500a94d2048c585faff6dDuncan Sands 2988c899ee031481dbece5f111379a274c848cb5902Duncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 29928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() == 30028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 30128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Scalar/Vector type mismatch"); 30228b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 30328b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 30428b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 3058c899ee031481dbece5f111379a274c848cb5902Duncan Sands SDValue LHS = GetScalarizedVector(N->getOperand(0)); 3068c899ee031481dbece5f111379a274c848cb5902Duncan Sands SDValue RHS = GetScalarizedVector(N->getOperand(1)); 307e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 3088c899ee031481dbece5f111379a274c848cb5902Duncan Sands 3098c899ee031481dbece5f111379a274c848cb5902Duncan Sands // Turn it into a scalar SETCC. 310825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 3118c899ee031481dbece5f111379a274c848cb5902Duncan Sands} 3128c899ee031481dbece5f111379a274c848cb5902Duncan Sands 313331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan SandsSDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 314e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 315331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands} 316331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands 317475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 31805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // Figure out if the scalar is the LHS or RHS and return it. 3190e3da1def46ae4925d811406f9e67f0c45d5b597Duncan Sands SDValue Arg = N->getOperand(2).getOperand(0); 3200e3da1def46ae4925d811406f9e67f0c45d5b597Duncan Sands if (Arg.getOpcode() == ISD::UNDEF) 321e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 3220e3da1def46ae4925d811406f9e67f0c45d5b597Duncan Sands unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 32305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands return GetScalarizedVector(N->getOperand(Op)); 32405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 32505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 326475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 32728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() && 32828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 32928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Operand types must be vectors"); 33028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 331475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue LHS = GetScalarizedVector(N->getOperand(0)); 332475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue RHS = GetScalarizedVector(N->getOperand(1)); 333e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT NVT = N->getValueType(0).getVectorElementType(); 334e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 335059b2dbd902233e1bbd040ec8c18c66b4186bb0eDuncan Sands 336059b2dbd902233e1bbd040ec8c18c66b4186bb0eDuncan Sands // Turn it into a scalar SETCC. 33728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 33828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(2)); 33928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands // Vectors may have a different boolean contents to scalars. Promote the 34028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands // value appropriately. 34128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands ISD::NodeType ExtendCode = 34228b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands TargetLowering::getExtendForContent(TLI.getBooleanContents(true)); 34328b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands return DAG.getNode(ExtendCode, DL, NVT, Res); 34411e56cb4dc73bbb0bdc083042657ea3a5aad63f2Duncan Sands} 34511e56cb4dc73bbb0bdc083042657ea3a5aad63f2Duncan Sands 34669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 34769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands//===----------------------------------------------------------------------===// 34869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands// Operand Vector Scalarization <1 x ty> -> ty. 34969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands//===----------------------------------------------------------------------===// 35069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 351f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sandsbool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 3527419b1806a1c3abdc23b62de76fae737b763fb33David Greene DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 3534437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 3547419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 355475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Res = SDValue(); 35669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 357ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (Res.getNode() == 0) { 35869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands switch (N->getOpcode()) { 35969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands default: 36069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands#ifndef NDEBUG 3617419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 3624437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 3637419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 36469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands#endif 365c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable("Do not know how to scalarize this operator's operand!"); 366bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: 367bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Res = ScalarizeVecOp_BITCAST(N); 3680a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner break; 36934fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach case ISD::ANY_EXTEND: 37034fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach case ISD::ZERO_EXTEND: 37134fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach case ISD::SIGN_EXTEND: 37234fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach Res = ScalarizeVecOp_EXTEND(N); 37334fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach break; 374d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::CONCAT_VECTORS: 3750a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner Res = ScalarizeVecOp_CONCAT_VECTORS(N); 3760a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner break; 37769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::EXTRACT_VECTOR_ELT: 3780a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 3790a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner break; 38069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands case ISD::STORE: 3810a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 3820a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner break; 38369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands } 38469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands } 38569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 38669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands // If the result is null, the sub-method took care of registering results etc. 387ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (!Res.getNode()) return false; 38869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 38947d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // If the result is N, the sub-method updated N in place. Tell the legalizer 39047d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // core about this. 39147d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands if (Res.getNode() == N) 39269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands return true; 39369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 39469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 39569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands "Invalid operand expansion"); 39669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 397475871a144eb604ddaf37503397ba0941442e5fbDan Gohman ReplaceValueWith(SDValue(N, 0), Res); 39869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands return false; 39969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 40069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 401bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck/// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 40269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands/// to be scalarized, it must be <1 x ty>. Convert the element instead. 403bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley PeckSDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 404475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Elt = GetScalarizedVector(N->getOperand(0)); 405ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BITCAST, SDLoc(N), 406644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen N->getValueType(0), Elt); 40769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 40869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 40934fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach/// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs 41034fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach/// to be scalarized, it must be <1 x ty>. Extend the element instead. 41134fd0d2b93edb3ddefe5c5766073273f86b23b78Jim GrosbachSDValue DAGTypeLegalizer::ScalarizeVecOp_EXTEND(SDNode *N) { 41234fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach assert(N->getValueType(0).getVectorNumElements() == 1 && 41334fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach "Unexected vector type!"); 41434fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach SDValue Elt = GetScalarizedVector(N->getOperand(0)); 41534fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach SmallVector<SDValue, 1> Ops(1); 416ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick Ops[0] = DAG.getNode(N->getOpcode(), SDLoc(N), 41734fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach N->getValueType(0).getScalarType(), Elt); 41834fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach // Revectorize the result so the types line up with what the uses of this 41934fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach // expression expect. 420ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), 42134fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach &Ops[0], 1); 42234fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach} 42334fd0d2b93edb3ddefe5c5766073273f86b23b78Jim Grosbach 424d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 425d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands/// use a BUILD_VECTOR instead. 426d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan SandsSDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 427d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands SmallVector<SDValue, 8> Ops(N->getNumOperands()); 428d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 429d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands Ops[i] = GetScalarizedVector(N->getOperand(i)); 430ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), 431a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng &Ops[0], Ops.size()); 432d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands} 433d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 43478cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 43578cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 43669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands/// index. 437475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 438c680ac90032bf455b2bba77de538fccea08eb267Eli Friedman SDValue Res = GetScalarizedVector(N->getOperand(0)); 439c680ac90032bf455b2bba77de538fccea08eb267Eli Friedman if (Res.getValueType() != N->getValueType(0)) 440ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), 441c680ac90032bf455b2bba77de538fccea08eb267Eli Friedman Res); 442c680ac90032bf455b2bba77de538fccea08eb267Eli Friedman return Res; 44369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 44469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 44578cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 44669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands/// scalarized, it must be <1 x ty>. Just store the element. 447475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 4487f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman assert(N->isUnindexed() && "Indexed store of one-element vector?"); 44969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 450e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 4517f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 4527f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman if (N->isTruncatingStore()) 453644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getTruncStore(N->getChain(), dl, 4547f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman GetScalarizedVector(N->getOperand(1)), 455da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner N->getBasePtr(), N->getPointerInfo(), 4567f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman N->getMemoryVT().getVectorElementType(), 4571e559443a17d1b335f697551c6263ba60d5dd827David Greene N->isVolatile(), N->isNonTemporal(), 4581e559443a17d1b335f697551c6263ba60d5dd827David Greene N->getAlignment()); 4597f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 460644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 461da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner N->getBasePtr(), N->getPointerInfo(), 4621e559443a17d1b335f697551c6263ba60d5dd827David Greene N->isVolatile(), N->isNonTemporal(), 4631e559443a17d1b335f697551c6263ba60d5dd827David Greene N->getOriginalAlignment()); 46469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands} 46569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 46669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 46713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 46813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// Result Vector Splitting 46913c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 47013c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner 471f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sands/// SplitVectorResult - This method is called when the specified result of the 47213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// specified node is found to need vector splitting. At this point, the node 47313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// may also have invalid operands or may have other results that need 47413c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// legalization, we just know that (at least) one result needs vector 47513c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// splitting. 476f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sandsvoid DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 4777419b1806a1c3abdc23b62de76fae737b763fb33David Greene DEBUG(dbgs() << "Split node result: "; 4784437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 4797419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 480475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Lo, Hi; 481f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper 482bbfa5c02be2ef2a9ff965a4dcfe9e99665dcf0efPete Cooper // See if the target wants to custom expand this node. 483bbfa5c02be2ef2a9ff965a4dcfe9e99665dcf0efPete Cooper if (CustomLowerNode(N, N->getValueType(ResNo), true)) 484bbfa5c02be2ef2a9ff965a4dcfe9e99665dcf0efPete Cooper return; 48569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 48613c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner switch (N->getOpcode()) { 48713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner default: 48813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner#ifndef NDEBUG 4897419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "SplitVectorResult #" << ResNo << ": "; 4904437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 4917419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 49213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner#endif 493200e04c3fdbdbdb671e29691ee201f34805e4dd1Craig Topper report_fatal_error("Do not know how to split the result of this " 494200e04c3fdbdbdb671e29691ee201f34805e4dd1Craig Topper "operator!\n"); 49569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 4964c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 49728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands case ISD::VSELECT: 49878cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 49978cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 50078cd649ad326f79a1f8424ca2b63cea3239a9a52Duncan Sands case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 501bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 502aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 503aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 504aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 505d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 506aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 50705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 508331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 509d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 5105962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner case ISD::LOAD: 5115962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 5125962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner break; 5135962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner case ISD::SETCC: 5145962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner SplitVecRes_SETCC(N, Lo, Hi); 5155962ed0a36bfad0fb591998964a0da7ddecda1f2Chris Lattner break; 5168c899ee031481dbece5f111379a274c848cb5902Duncan Sands case ISD::VECTOR_SHUFFLE: 5178c899ee031481dbece5f111379a274c848cb5902Duncan Sands SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 5188c899ee031481dbece5f111379a274c848cb5902Duncan Sands break; 51905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 520e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::ANY_EXTEND: 521b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman case ISD::CONVERT_RNDSAT: 522eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::CTLZ: 523e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::CTTZ: 52463974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTLZ_ZERO_UNDEF: 52563974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTTZ_ZERO_UNDEF: 52663974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTPOP: 527eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::FABS: 528e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FCEIL: 529eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::FCOS: 530e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FEXP: 531e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FEXP2: 532509e84fa7146175c86dec5ef2167290f294dc89eDan Gohman case ISD::FFLOOR: 533e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG: 534e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG10: 535e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG2: 536509e84fa7146175c86dec5ef2167290f294dc89eDan Gohman case ISD::FNEARBYINT: 537e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FNEG: 5389c45251e1165a9ed8c351468ebb01b3859ea1df3Duncan Sands case ISD::FP_EXTEND: 539b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman case ISD::FP_ROUND: 540eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::FP_TO_SINT: 541eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::FP_TO_UINT: 542e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FRINT: 543e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FSIN: 544e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FSQRT: 545e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FTRUNC: 546e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::SIGN_EXTEND: 547eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner case ISD::SINT_TO_FP: 548d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::TRUNCATE: 549e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::UINT_TO_FP: 55064a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ZERO_EXTEND: 55164a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner SplitVecRes_UnaryOp(N, Lo, Hi); 55264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 55305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 554697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::ADD: 555697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::SUB: 556697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::MUL: 557697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::FADD: 558697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::FSUB: 559697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::FMUL: 560697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::SDIV: 561697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::UDIV: 562697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::FDIV: 563697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::FPOW: 564697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::AND: 565697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::OR: 566697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::XOR: 56793b3b928d70df956b81ab1bffb7e8bab4b0c8fd7Mon P Wang case ISD::SHL: 56893b3b928d70df956b81ab1bffb7e8bab4b0c8fd7Mon P Wang case ISD::SRA: 569ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands case ISD::SRL: 570697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::UREM: 571697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner case ISD::SREM: 57264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::FREM: 57364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner SplitVecRes_BinOp(N, Lo, Hi); 57464a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 575b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper case ISD::FMA: 576b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SplitVecRes_TernaryOp(N, Lo, Hi); 577b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper break; 57813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner } 57969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 58013c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner // If Lo/Hi is null, the sub-method took care of registering results etc. 581ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (Lo.getNode()) 582475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SetSplitVector(SDValue(N, ResNo), Lo, Hi); 58313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner} 58413c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner 585475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 586475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 587475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue LHSLo, LHSHi; 58805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 589475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue RHSLo, RHSHi; 59005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 591e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 59205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 593644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 594644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 59505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 59605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 597b49998d76cb4e414d13d60116adf13b085d85dc1Pete Coopervoid DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 598b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue &Hi) { 599b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op0Lo, Op0Hi; 600b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 601b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op1Lo, Op1Hi; 602b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 603b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper SDValue Op2Lo, Op2Hi; 604b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 605e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 606b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper 607b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 608b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper Op0Lo, Op1Lo, Op2Lo); 609b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 610b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper Op0Hi, Op1Hi, Op2Hi); 611b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper} 612b49998d76cb4e414d13d60116adf13b085d85dc1Pete Cooper 613bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peckvoid DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 614bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck SDValue &Hi) { 61505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // We know the result is a vector. The input may be either a vector or a 61605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // scalar value. 617e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 61805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 619e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 62069b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 621475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue InOp = N->getOperand(0); 622e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 62369b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 62405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // Handle some special cases efficiently. 62505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands switch (getTypeAction(InVT)) { 62696e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeLegal: 62796e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypePromoteInteger: 62896e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeSoftenFloat: 62996e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeScalarizeVector: 63024f05334e60bd2b2b614d5eb72ee6c92ad0a9bc8Eli Friedman case TargetLowering::TypeWidenVector: 63105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands break; 63296e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeExpandInteger: 63396e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeExpandFloat: 63405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // A scalar to vector conversion, where the scalar needs expansion. 63505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // If the vector is being split in two then we can just convert the 63605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // expanded pieces. 63705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (LoVT == HiVT) { 63805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetExpandedOp(InOp, Lo, Hi); 63905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (TLI.isBigEndian()) 64005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands std::swap(Lo, Hi); 641bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 642bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 64305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands return; 64405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands } 64505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands break; 64696e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeSplitVector: 64705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // If the input is a vector that needs to be split, convert each split 64805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // piece of the input now. 64905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(InOp, Lo, Hi); 650bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 651bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 65205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands return; 65305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands } 6548cc364c5cc210c56808011a5ebad28a719f106b7Duncan Sands 65505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // In the general case, convert the input to an integer and split it by hand. 65623b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 65723b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 65805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (TLI.isBigEndian()) 65905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands std::swap(LoIntVT, HiIntVT); 6608cc364c5cc210c56808011a5ebad28a719f106b7Duncan Sands 66105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 66205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 66305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (TLI.isBigEndian()) 66405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands std::swap(Lo, Hi); 665bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 666bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 66705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 66805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 669475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 670475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 671e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 672e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 67305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 67405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands unsigned LoNumElts = LoVT.getVectorNumElements(); 675475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 676a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 67705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 678475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 679a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 68005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 68105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 682475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 683475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 68405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 685e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 68605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands unsigned NumSubvectors = N->getNumOperands() / 2; 68705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (NumSubvectors == 1) { 68805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands Lo = N->getOperand(0); 68905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands Hi = N->getOperand(1); 69005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands return; 6918cc364c5cc210c56808011a5ebad28a719f106b7Duncan Sands } 69269b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 693e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 69405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 69505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 696475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 697644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 69805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 699475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 700644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 70105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 70205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 703aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wangvoid DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 704aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang SDValue &Hi) { 7051e465a35c8cd3d2ddd5e1d15fca7ffd3a8dbb565Duncan Sands SDValue Vec = N->getOperand(0); 7061e465a35c8cd3d2ddd5e1d15fca7ffd3a8dbb565Duncan Sands SDValue Idx = N->getOperand(1); 707e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 7081e465a35c8cd3d2ddd5e1d15fca7ffd3a8dbb565Duncan Sands 709e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 710aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 711aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang 712644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 7136736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 7146736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 715425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), 716425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 717aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang} 718aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang 719475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 720475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 721e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 72205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(N->getOperand(0), Lo, Hi); 723644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 724644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 725697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner} 726697b53e0b2278ce39974748074fc23014a6636c5Chris Lattner 727d1996360399ad6dbe75ee185b661b16c83146373Dan Gohmanvoid DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 728d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman SDValue &Hi) { 729d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman SDValue LHSLo, LHSHi; 730d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 731e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 732d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman 733d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman EVT LoVT, HiVT; 734d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 735d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman 736d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 737d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman DAG.getValueType(LoVT)); 738d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 739d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman DAG.getValueType(HiVT)); 740d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman} 741d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman 742475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 743475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 744475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Vec = N->getOperand(0); 745475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Elt = N->getOperand(1); 746475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Idx = N->getOperand(2); 747e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 74869b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands GetSplitVector(Vec, Lo, Hi); 7497d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands 7507d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 751f5aeb1a8e4cf272c7348376d185ef8d8267653e0Dan Gohman unsigned IdxVal = CIdx->getZExtValue(); 7527d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 7537d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands if (IdxVal < LoNumElts) 754644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 755644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo.getValueType(), Lo, Elt, Idx); 7567d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands else 757644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 758425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(IdxVal - LoNumElts, 759425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 7607d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands return; 7617d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands } 7627d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands 7637d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands // Spill the vector to the stack. 764e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VecVT = Vec.getValueType(); 765e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VecVT.getVectorElementType(); 766475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 767ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 768ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner MachinePointerInfo(), false, false, 0); 7697d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands 770262e04b3078e38c8dc5cdc40bf51e7cd467fd57aDuncan Sands // Store the new element. This may be larger than the vector element type, 771262e04b3078e38c8dc5cdc40bf51e7cd467fd57aDuncan Sands // so use a truncating store. 772475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 773db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 77447d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands unsigned Alignment = 7753574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow TLI.getDataLayout()->getPrefTypeAlignment(VecType); 776ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 7771e559443a17d1b335f697551c6263ba60d5dd827David Greene false, false, 0); 7787d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands 77947d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // Load the Lo part from the stack slot. 780ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 781d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper false, false, false, 0); 78247d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands 78347d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // Increment the pointer to the other part. 784aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 785644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 78647d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands DAG.getIntPtrConstant(IncrementSize)); 7877d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands 78847d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // Load the Hi part from the stack slot. 789ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 790d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper false, false, false, MinAlign(Alignment, IncrementSize)); 791eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner} 792eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner 793331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sandsvoid DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 794331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands SDValue &Hi) { 795e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 796e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 797331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 798644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 799e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen Hi = DAG.getUNDEF(HiVT); 800331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands} 801331a746101aff2199c19e5a7407a5ca6a4bbdafaDuncan Sands 802475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 803475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 80405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 805e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 806e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(LD); 80705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 80805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 8097f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman ISD::LoadExtType ExtType = LD->getExtensionType(); 810475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Ch = LD->getChain(); 811475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Ptr = LD->getBasePtr(); 812e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 813e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT MemoryVT = LD->getMemoryVT(); 814101b25c028706c61c7dd9fb92d0b3c1541cb12b6Nate Begeman unsigned Alignment = LD->getOriginalAlignment(); 81505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands bool isVolatile = LD->isVolatile(); 8161e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNonTemporal = LD->isNonTemporal(); 817d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper bool isInvariant = LD->isInvariant(); 81805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 819e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoMemVT, HiMemVT; 8207f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 82105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 822bcc8017c738e92d9c1af221b11c4916cb524184eEvan Cheng Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 8235cd9391a089d146d8a18ade602769502e5a8d36fChris Lattner LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 824d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper isInvariant, Alignment); 82505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 826aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 827644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 8287f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman DAG.getIntPtrConstant(IncrementSize)); 829bcc8017c738e92d9c1af221b11c4916cb524184eEvan Cheng Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 8305cd9391a089d146d8a18ade602769502e5a8d36fChris Lattner LD->getPointerInfo().getWithOffset(IncrementSize), 831d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment); 8327f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 8337f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman // Build a factor node to remember that this load is independent of the 8347f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman // other one. 835825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 8367f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman Hi.getValue(1)); 83705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 83805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // Legalized the chain result - switch anything that used the old chain to 83905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // use the new one. 840475871a144eb604ddaf37503397ba0941442e5fbDan Gohman ReplaceValueWith(SDValue(LD, 1), Ch); 84105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 84205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 8438c899ee031481dbece5f111379a274c848cb5902Duncan Sandsvoid DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 84428b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() && 84528b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 84628b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Operand types must be vectors"); 84728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 848e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 849e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 8508c899ee031481dbece5f111379a274c848cb5902Duncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 8518c899ee031481dbece5f111379a274c848cb5902Duncan Sands 8528c899ee031481dbece5f111379a274c848cb5902Duncan Sands // Split the input. 853e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = N->getOperand(0).getValueType(); 8548c899ee031481dbece5f111379a274c848cb5902Duncan Sands SDValue LL, LH, RL, RH; 85523b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 856f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands LoVT.getVectorNumElements()); 857f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 858425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 859f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 860425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(InNVT.getVectorNumElements(), 861425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 862f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands 863f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 864425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 865f2d754bb382cba0bad2774144ddac84be5354d16Duncan Sands RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 866425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(InNVT.getVectorNumElements(), 867425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 8688c899ee031481dbece5f111379a274c848cb5902Duncan Sands 8698c899ee031481dbece5f111379a274c848cb5902Duncan Sands Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 8708c899ee031481dbece5f111379a274c848cb5902Duncan Sands Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 8718c899ee031481dbece5f111379a274c848cb5902Duncan Sands} 8728c899ee031481dbece5f111379a274c848cb5902Duncan Sands 873475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 874475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue &Hi) { 8754ddc41e58cae9236b7959cbed62a5a052f05e70eDuncan Sands // Get the dest types - they may not match the input types, e.g. int_to_fp. 876e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoVT, HiVT; 877e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 87805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 87905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 880a921a468542a804ccebb680935175798ac48868bDuncan Sands // If the input also splits, handle it directly for a compile time speedup. 881a921a468542a804ccebb680935175798ac48868bDuncan Sands // Otherwise split it by hand. 882e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = N->getOperand(0).getValueType(); 883a921a468542a804ccebb680935175798ac48868bDuncan Sands if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 884a921a468542a804ccebb680935175798ac48868bDuncan Sands GetSplitVector(N->getOperand(0), Lo, Hi); 885a921a468542a804ccebb680935175798ac48868bDuncan Sands } else { 88623b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 8871c11debc94baa65b7af702bbc20c813c266afb19Duncan Sands LoVT.getVectorNumElements()); 888644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 889425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 890644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 891425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(InNVT.getVectorNumElements(), 892425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 8931c11debc94baa65b7af702bbc20c813c266afb19Duncan Sands } 8941c11debc94baa65b7af702bbc20c813c266afb19Duncan Sands 895b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman if (N->getOpcode() == ISD::FP_ROUND) { 896b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 897b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 898b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 899b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue DTyOpLo = DAG.getValueType(LoVT); 900b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue DTyOpHi = DAG.getValueType(HiVT); 901b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 902b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 903b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue RndOp = N->getOperand(3); 904b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman SDValue SatOp = N->getOperand(4); 905b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 906b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 907b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman CvtCode); 908b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 909b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman CvtCode); 910b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman } else { 911b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 912b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 913b141099c14bfa86167b2137e8a9544c6ee805955Eli Friedman } 91405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 91505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 9165a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begemanvoid DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 9175a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman SDValue &Lo, SDValue &Hi) { 918c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The low and high parts of the original input give four input vectors. 919c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands SDValue Inputs[4]; 920e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 921c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 922c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 923e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT NewVT = Inputs[0].getValueType(); 924c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands unsigned NewElts = NewVT.getVectorNumElements(); 925c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 926c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // If Lo or Hi uses elements from at most two of the four input vectors, then 927c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // express it as a vector shuffle of those two inputs. Otherwise extract the 928c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 9299008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman SmallVector<int, 16> Ops; 930c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands for (unsigned High = 0; High < 2; ++High) { 931c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands SDValue &Output = High ? Hi : Lo; 932c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 933c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Build a shuffle mask for the output, discovering on the fly which 934c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // input vectors to use as shuffle operands (recorded in InputUsed). 935c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // If building a suitable shuffle vector proves too hard, then bail 936c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // out with useBuildVector set. 937c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 938c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands unsigned FirstMaskIdx = High * NewElts; 939c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands bool useBuildVector = false; 940c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 941c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The mask element. This indexes into the input. 9425a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 943c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 944c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The input vector this mask element indexes into. 9459008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman unsigned Input = (unsigned)Idx / NewElts; 946c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 947c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands if (Input >= array_lengthof(Inputs)) { 948c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The mask element does not index into any input vector. 9499008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman Ops.push_back(-1); 950c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands continue; 951c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } 952c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 953c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Turn the index into an offset from the start of the input vector. 954c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands Idx -= Input * NewElts; 955c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 956c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Find or create a shuffle vector operand to hold this input. 957c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands unsigned OpNo; 958c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 959c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands if (InputUsed[OpNo] == Input) { 960c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // This input vector is already an operand. 961c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands break; 962c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } else if (InputUsed[OpNo] == -1U) { 963c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Create a new operand for this input vector. 964c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands InputUsed[OpNo] = Input; 965c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands break; 966c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } 967c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } 968c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 969c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands if (OpNo >= array_lengthof(InputUsed)) { 970c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // More than two input vectors used! Give up on trying to create a 971c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 972c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands useBuildVector = true; 973c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands break; 9740e3da1def46ae4925d811406f9e67f0c45d5b597Duncan Sands } 975c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 976c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Add the mask index for the new shuffle vector. 9779008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman Ops.push_back(Idx + OpNo * NewElts); 978eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner } 97969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 980c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands if (useBuildVector) { 981e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = NewVT.getVectorElementType(); 9829008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman SmallVector<SDValue, 16> SVOps; 983c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 984c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Extract the input elements by hand. 985c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 986c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The mask element. This indexes into the input. 9875a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 988c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 989c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The input vector this mask element indexes into. 9909008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman unsigned Input = (unsigned)Idx / NewElts; 991c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 992c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands if (Input >= array_lengthof(Inputs)) { 993c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // The mask element is "undef" or indexes off the end of the input. 9949008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman SVOps.push_back(DAG.getUNDEF(EltVT)); 995c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands continue; 996c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } 997c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 998c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Turn the index into an offset from the start of the input vector. 999c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands Idx -= Input * NewElts; 1000c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 1001c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Extract the vector element by hand. 10029008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 1003425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard Inputs[Input], DAG.getConstant(Idx, 1004425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy()))); 1005aeb06d246254e4829a49164a11eacced9a43d9d4Mon P Wang } 1006c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 1007c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // Construct the Lo/Hi output using a BUILD_VECTOR. 10089008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 1009c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } else if (InputUsed[0] == -1U) { 1010c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // No input vectors were used! The result is undefined. 1011e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen Output = DAG.getUNDEF(NewVT); 1012c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands } else { 1013c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands SDValue Op0 = Inputs[InputUsed[0]]; 1014c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands // If only one input was used, use an undefined vector for the other. 1015c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands SDValue Op1 = InputUsed[1] == -1U ? 1016e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 10179008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman // At least one input vector was used. Create a new shuffle vector. 10189008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1019eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner } 1020c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands 1021c529168f29a9821806af8c7096fdcfbec06343d3Duncan Sands Ops.clear(); 1022eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner } 1023eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner} 1024eeaad40246e45ec48c85acac4ab57e82457abf19Chris Lattner 102511e56cb4dc73bbb0bdc083042657ea3a5aad63f2Duncan Sands 102613c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 102713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner// Operand Vector Splitting 102813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner//===----------------------------------------------------------------------===// 102913c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner 1030f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sands/// SplitVectorOperand - This method is called when the specified operand of the 103113c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// specified node is found to need vector splitting. At this point, all of the 103213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// result types of the node are known to be legal, but other operands of the 103313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner/// node may need legalization as well as the specified one. 1034f4e4629ee8c218f892ad8ae3e182fe40bc160895Duncan Sandsbool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 10357419b1806a1c3abdc23b62de76fae737b763fb33David Greene DEBUG(dbgs() << "Split node operand: "; 10364437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 10377419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 1038475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Res = SDValue(); 103969b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 1040320185fa5f5838b3892962f6e91083e9729cd946Justin Holewinski // See if the target wants to custom split this node. 1041320185fa5f5838b3892962f6e91083e9729cd946Justin Holewinski if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1042320185fa5f5838b3892962f6e91083e9729cd946Justin Holewinski return false; 1043320185fa5f5838b3892962f6e91083e9729cd946Justin Holewinski 1044ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (Res.getNode() == 0) { 104513c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner switch (N->getOpcode()) { 104613c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner default: 104713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner#ifndef NDEBUG 10487419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 10494437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 10507419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 105113c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner#endif 1052ea0ca846478b8296f679afa419f2ad4e28073ad5Craig Topper report_fatal_error("Do not know how to split this operator's " 1053ea0ca846478b8296f679afa419f2ad4e28073ad5Craig Topper "operand!\n"); 1054ea0ca846478b8296f679afa419f2ad4e28073ad5Craig Topper 105528b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1056bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 105705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 105805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 10598306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 10600cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break; 1061593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 106264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::STORE: 106364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 106464a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 10657f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski case ISD::VSELECT: 10667f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski Res = SplitVecOp_VSELECT(N, OpNo); 10677f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski break; 1068d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::CTTZ: 1069d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::CTLZ: 1070d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::CTPOP: 1071e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FP_EXTEND: 1072d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::FP_TO_SINT: 1073d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::FP_TO_UINT: 1074d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands case ISD::SINT_TO_FP: 107564a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::UINT_TO_FP: 1076593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner case ISD::FTRUNC: 107764a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::SIGN_EXTEND: 107864a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ZERO_EXTEND: 107964a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ANY_EXTEND: 108064a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = SplitVecOp_UnaryOp(N); 108164a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 108213c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner } 108313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner } 108469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 108513c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner // If the result is null, the sub-method took care of registering results etc. 1086ba36cb5242eb02b12b277f82b9efe497f7da4d7fGabor Greif if (!Res.getNode()) return false; 108769b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 108847d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // If the result is N, the sub-method updated N in place. Tell the legalizer 108947d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands // core about this. 109047d9dcc584cdb7fd645ca1d5c2a0ce363570aeb7Duncan Sands if (Res.getNode() == N) 109113c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner return true; 1092212a11c417e272cc8fd12e66cfe5110c47559e17Duncan Sands 109313c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 109413c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner "Invalid operand expansion"); 109569b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands 1096475871a144eb604ddaf37503397ba0941442e5fbDan Gohman ReplaceValueWith(SDValue(N, 0), Res); 109713c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner return false; 109813c6a1740cb8877f10e202ee1442231e0c4a903aChris Lattner} 109950187864c12dbf15f021cacfd6922d9c014f3f8dChris Lattner 11007f128ea00c5358729906a9b98f844e887a1c3d73Justin HolewinskiSDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 11017f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski // The only possibility for an illegal operand is the mask, since result type 11027f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski // legalization would have handled this node already otherwise. 11037f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski assert(OpNo == 0 && "Illegal operand must be mask"); 11047f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski 11057f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski SDValue Mask = N->getOperand(0); 11067f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski SDValue Src0 = N->getOperand(1); 11077f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski SDValue Src1 = N->getOperand(2); 1108e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 11097f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski EVT MaskVT = Mask.getValueType(); 11107f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski assert(MaskVT.isVector() && "VSELECT without a vector mask?"); 11117f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski 11127f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski SDValue Lo, Hi; 11137f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski GetSplitVector(N->getOperand(0), Lo, Hi); 1114db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski assert(Lo.getValueType() == Hi.getValueType() && 1115e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer "Lo and Hi have differing types"); 11167f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski 11177f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 11187f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski unsigned HiNumElts = Hi.getValueType().getVectorNumElements(); 11197f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski assert(LoNumElts == HiNumElts && "Asymmetric vector split?"); 11207f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski 1121db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski LLVMContext &Ctx = *DAG.getContext(); 1122425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard SDValue Zero = DAG.getConstant(0, TLI.getVectorIdxTy()); 1123425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard SDValue LoElts = DAG.getConstant(LoNumElts, TLI.getVectorIdxTy()); 1124db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT Src0VT = Src0.getValueType(); 1125db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT Src0EltTy = Src0VT.getVectorElementType(); 1126db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT MaskEltTy = MaskVT.getVectorElementType(); 1127db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1128db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT LoOpVT = EVT::getVectorVT(Ctx, Src0EltTy, LoNumElts); 1129db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT LoMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, LoNumElts); 1130db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT HiOpVT = EVT::getVectorVT(Ctx, Src0EltTy, HiNumElts); 1131db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski EVT HiMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, HiNumElts); 1132db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1133db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue LoOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src0, Zero); 1134db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue LoOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src1, Zero); 1135db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1136db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue HiOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src0, LoElts); 1137db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue HiOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src1, LoElts); 1138db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1139db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue LoMask = 1140db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoMaskVT, Mask, Zero); 1141db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue HiMask = 1142db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiMaskVT, Mask, LoElts); 1143db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1144db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue LoSelect = 1145db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1146db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski SDValue HiSelect = 1147db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1148db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski 1149db62a883a78cef4ff63a699452368fefa4b3fefdJustin Holewinski return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 11507f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski} 11517f128ea00c5358729906a9b98f844e887a1c3d73Justin Holewinski 1152d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan SandsSDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1153d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands // The result has a legal vector type, but the input needs splitting. 1154e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT ResVT = N->getValueType(0); 1155d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands SDValue Lo, Hi; 1156e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 1157d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands GetSplitVector(N->getOperand(0), Lo, Hi); 1158e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = Lo.getValueType(); 1159d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 116023b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1161d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands InVT.getVectorNumElements()); 1162d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 1163644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1164644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1165d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 1166644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1167d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands} 1168d398672dddc50704ba7c4adbcf7dc99bd14c0108Duncan Sands 1169bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley PeckSDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1170bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1171077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands // end up being split all the way down to individual components. Convert the 1172077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands // split pieces into integers and reassemble. 1173475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Lo, Hi; 117469b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands GetSplitVector(N->getOperand(0), Lo, Hi); 1175d8742eeb2f7cabc45a1c3736a2780bf87ba684baDuncan Sands Lo = BitConvertToInteger(Lo); 1176d8742eeb2f7cabc45a1c3736a2780bf87ba684baDuncan Sands Hi = BitConvertToInteger(Hi); 1177077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1178077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands if (TLI.isBigEndian()) 1179077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands std::swap(Lo, Hi); 1180077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1181ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 1182ac7613a3263caa80d735f3fbf2b9f7b81deabc08Duncan Sands JoinIntegers(Lo, Hi)); 1183077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands} 1184077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1185475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 11866736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson // We know that the extracted result type is legal. 1187e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT SubVT = N->getValueType(0); 1188475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Idx = N->getOperand(1); 1189e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 1190475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Lo, Hi; 119105c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(N->getOperand(0), Lo, Hi); 119205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 119305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1194f5aeb1a8e4cf272c7348376d185ef8d8267653e0Dan Gohman uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 119505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 119605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands if (IdxVal < LoElts) { 119705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 119805c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands "Extracted subvector crosses vector split!"); 1199644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 120005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands } else { 1201644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 120205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 120305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands } 120405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands} 120505c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 1206475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1207475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Vec = N->getOperand(0); 1208475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Idx = N->getOperand(1); 1209e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VecVT = Vec.getValueType(); 1210077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1211077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands if (isa<ConstantSDNode>(Idx)) { 1212f5aeb1a8e4cf272c7348376d185ef8d8267653e0Dan Gohman uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 121383ec4b6711980242ef3c55a4fa36b2d7a39c1bfbDuncan Sands assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1214077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1215475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Lo, Hi; 121669b01e92a29ce6d7e435171aeea3fbc987b81586Duncan Sands GetSplitVector(Vec, Lo, Hi); 1217077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 121883ec4b6711980242ef3c55a4fa36b2d7a39c1bfbDuncan Sands uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1219077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1220077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands if (IdxVal < LoElts) 1221027657db7cf60bcbf40403496d7e4a170f9ce1ecDan Gohman return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1222027657db7cf60bcbf40403496d7e4a170f9ce1ecDan Gohman return SDValue(DAG.UpdateNodeOperands(N, Hi, 12230a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner DAG.getConstant(IdxVal - LoElts, 12248306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner Idx.getValueType())), 0); 1225077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands } 1226077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 12277d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands // Store the vector to the stack. 1228e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VecVT.getVectorElementType(); 1229e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 1230475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 12316229d0acb8f395552131a7015a5d1e7b2bae2111Chris Lattner SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 12326229d0acb8f395552131a7015a5d1e7b2bae2111Chris Lattner MachinePointerInfo(), false, false, 0); 1233077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 12347d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands // Load back the required element. 12357d0d8460646d1a06ff561775d40123a4cf65bf4dDuncan Sands StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1236a901129169194881a78b7fd8953e09f55b846d10Stuart Hastings return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 12376229d0acb8f395552131a7015a5d1e7b2bae2111Chris Lattner MachinePointerInfo(), EltVT, false, false, 0); 1238077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands} 1239077f9b20d0e8659d00a09046a63e28edf0665ffeDuncan Sands 1240475871a144eb604ddaf37503397ba0941442e5fbDan GohmanSDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 12417f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman assert(N->isUnindexed() && "Indexed store of vector?"); 124205c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands assert(OpNo == 1 && "Can only split the stored value"); 1243e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 124405c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 12457f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman bool isTruncating = N->isTruncatingStore(); 1246475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Ch = N->getChain(); 1247475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Ptr = N->getBasePtr(); 1248e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT MemoryVT = N->getMemoryVT(); 1249101b25c028706c61c7dd9fb92d0b3c1541cb12b6Nate Begeman unsigned Alignment = N->getOriginalAlignment(); 125005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands bool isVol = N->isVolatile(); 12511e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNT = N->isNonTemporal(); 1252475871a144eb604ddaf37503397ba0941442e5fbDan Gohman SDValue Lo, Hi; 125305c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands GetSplitVector(N->getOperand(1), Lo, Hi); 1254f83b1f63ddf27aaba791393940f37709ebbda33bDuncan Sands 1255e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT LoMemVT, HiMemVT; 12567f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1257f83b1f63ddf27aaba791393940f37709ebbda33bDuncan Sands 1258aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 12597f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 12607f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman if (isTruncating) 1261da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 12621e559443a17d1b335f697551c6263ba60d5dd827David Greene LoMemVT, isVol, isNT, Alignment); 12637f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman else 1264da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 12651e559443a17d1b335f697551c6263ba60d5dd827David Greene isVol, isNT, Alignment); 126605c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 126705c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands // Increment the pointer to the other half. 12688306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 126905c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands DAG.getIntPtrConstant(IncrementSize)); 127005c397d52a145c8844790d6491c4c51d4bbfed7cDuncan Sands 12717f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman if (isTruncating) 1272da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1273da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner N->getPointerInfo().getWithOffset(IncrementSize), 12741e559443a17d1b335f697551c6263ba60d5dd827David Greene HiMemVT, isVol, isNT, Alignment); 12757f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman else 1276da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1277da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner N->getPointerInfo().getWithOffset(IncrementSize), 12781e559443a17d1b335f697551c6263ba60d5dd827David Greene isVol, isNT, Alignment); 12797f8613e5b8398b688080e3c944ab8c11593e1ed0Dan Gohman 12808306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 12818306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner} 12828306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner 12838306968c147d5861d8a53fba86ac0fbf5c050b84Chris LattnerSDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1284e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 1285bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 12861e48093df8971ba188d943d71eb7cdb1fb65aa42Jim Grosbach // The input operands all must have the same type, and we know the result 12871e48093df8971ba188d943d71eb7cdb1fb65aa42Jim Grosbach // type is valid. Convert this to a buildvector which extracts all the 12888306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // input elements. 12898306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // TODO: If the input elements are power-two vectors, we could convert this to 12908306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // a new CONCAT_VECTORS node with elements that are half-wide. 12918306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SmallVector<SDValue, 32> Elts; 12928306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner EVT EltVT = N->getValueType(0).getVectorElementType(); 12938306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 12948306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SDValue Op = N->getOperand(op); 12958306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 12968306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner i != e; ++i) { 12978306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1298425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard Op, DAG.getConstant(i, TLI.getVectorIdxTy()))); 12998306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner 13008306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 13018306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 1302bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 13038306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), 13048306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner &Elts[0], Elts.size()); 1305f83b1f63ddf27aaba791393940f37709ebbda33bDuncan Sands} 13068745b52853c9b1d1370005d7cb42587ebe2a7193Duncan Sands 13070cb1019e9cd41237408eae09623eb9a34a4cbe0cJim GrosbachSDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { 13080cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // The result type is legal, but the input type is illegal. If splitting 13090cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // ends up with the result type of each half still being legal, just 13100cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // do that. If, however, that would result in an illegal result type, 13110cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // we can try to get more clever with power-two vectors. Specifically, 13120cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // split the input type, but also widen the result element size, then 13130cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // concatenate the halves and truncate again. For example, consider a target 13140cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 13150cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 13160cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %inlo = v4i32 extract_subvector %in, 0 13170cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %inhi = v4i32 extract_subvector %in, 4 13180cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %lo16 = v4i16 trunc v4i32 %inlo 13190cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %hi16 = v4i16 trunc v4i32 %inhi 13200cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 13210cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // %res = v8i8 trunc v8i16 %in16 13220cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // 13230cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Without this transform, the original truncate would end up being 13240cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // scalarized, which is pretty much always a last resort. 13250cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue InVec = N->getOperand(0); 13260cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT InVT = InVec->getValueType(0); 13270cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT OutVT = N->getValueType(0); 13280cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach unsigned NumElements = OutVT.getVectorNumElements(); 13290cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Widening should have already made sure this is a power-two vector 13300cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // if we're trying to split it at all. assert() that's true, just in case. 13310cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 13320cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach 13330cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 13340cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 13350cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach 13360cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // If the input elements are only 1/2 the width of the result elements, 13370cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // just use the normal splitting. Our trick only work if there's room 13380cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // to split more than once. 13390cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach if (InElementSize <= OutElementSize * 2) 13400cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach return SplitVecOp_UnaryOp(N); 1341e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 13420cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach 13430cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Extract the halves of the input via extract_subvector. 13440cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), 13450cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach InVT.getVectorElementType(), NumElements/2); 13460cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue InLoVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec, 1347425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 13480cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue InHiVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec, 1349425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(NumElements/2, 1350425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 13510cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Truncate them to 1/2 the element size. 13520cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 13530cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 13540cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach NumElements/2); 13550cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec); 13560cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec); 13570cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Concatenate them to get the full intermediate truncation result. 13580cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 13590cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 13600cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach HalfHi); 13610cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // Now finish up by truncating all the way down to the original result 13620cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // type. This should normally be something that ends up being legal directly, 13630cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // but in theory if a target has very wide vectors and an annoyingly 13640cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach // restricted set of legal types, this split can chain to build things up. 13650cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 13660cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach} 13670cb1019e9cd41237408eae09623eb9a34a4cbe0cJim Grosbach 136828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan SandsSDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 136928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() && 137028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 137128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Operand types must be vectors"); 137228b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands // The result has a legal vector type, but the input needs splitting. 137328b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1374e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 137528b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands GetSplitVector(N->getOperand(0), Lo0, Hi0); 137628b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands GetSplitVector(N->getOperand(1), Lo1, Hi1); 137728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 137828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 137928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 138028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 138128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 138228b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 138328b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 138428b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands return PromoteTargetBoolean(Con, N->getValueType(0)); 138528b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands} 138628b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 138728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 1388593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris LattnerSDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1389593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner // The result has a legal vector type, but the input needs splitting. 1390593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner EVT ResVT = N->getValueType(0); 1391593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner SDValue Lo, Hi; 1392e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 1393593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner GetSplitVector(N->getOperand(0), Lo, Hi); 1394593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner EVT InVT = Lo.getValueType(); 1395f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper 1396593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1397593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner InVT.getVectorNumElements()); 1398f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper 1399593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1400593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1401f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper 1402593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1403f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper} 1404593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner 1405593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner 140687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 140787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 140887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang// Result Vector Widening 140987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 141087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 141187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wangvoid DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 14127419b1806a1c3abdc23b62de76fae737b763fb33David Greene DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 14134437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 14147419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 141587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1416cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang // See if the target wants to custom widen this node. 1417cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1418cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang return; 1419cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang 1420cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang SDValue Res = SDValue(); 142187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang switch (N->getOpcode()) { 142287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang default: 142387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang#ifndef NDEBUG 14247419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "WidenVectorResult #" << ResNo << ": "; 14254437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 14267419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 142787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang#endif 1428c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable("Do not know how to widen the result of this operator!"); 142987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 14304c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1431bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 143287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 143387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 143487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 143587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1436d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 143787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 143887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 143987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1440d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1441ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem case ISD::VSELECT: 144287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 144387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 14446fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 144587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 14468c899ee031481dbece5f111379a274c848cb5902Duncan Sands case ISD::VECTOR_SHUFFLE: 14470a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 14480a6c2d8deab5124198e8cd5fbb3c43509be3e511Chris Lattner break; 144987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::ADD: 145087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::AND: 145187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::BSWAP: 145287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FADD: 145387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FCOPYSIGN: 145487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FDIV: 145587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FMUL: 145687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FPOW: 145787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FREM: 145887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FSUB: 145987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::MUL: 146087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::MULHS: 146187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::MULHU: 146287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::OR: 146387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SDIV: 146487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SREM: 146587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::UDIV: 146687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::UREM: 146787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SUB: 146864a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::XOR: 146964a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = WidenVecRes_Binary(N); 147064a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 147187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1472956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman case ISD::FPOWI: 1473956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman Res = WidenVecRes_POWI(N); 1474956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman break; 1475956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman 147687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SHL: 147787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SRA: 147864a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::SRL: 147964a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = WidenVecRes_Shift(N); 148064a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 148187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1482e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::ANY_EXTEND: 1483e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FP_EXTEND: 148487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FP_ROUND: 148587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FP_TO_SINT: 148687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FP_TO_UINT: 1487e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::SIGN_EXTEND: 148887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SINT_TO_FP: 148987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::TRUNCATE: 1490e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::UINT_TO_FP: 149187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::ZERO_EXTEND: 149264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = WidenVecRes_Convert(N); 149364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 149487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 149587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CTLZ: 149687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CTPOP: 149787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CTTZ: 149887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FABS: 1499e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FCEIL: 150087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FCOS: 1501956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman case ISD::FEXP: 1502956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman case ISD::FEXP2: 1503e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FFLOOR: 1504956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman case ISD::FLOG: 1505956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman case ISD::FLOG10: 1506e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FLOG2: 1507e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FNEARBYINT: 1508e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FNEG: 1509e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FRINT: 1510e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FSIN: 1511e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FSQRT: 1512e80338af3f74089ab9fccd9bfb9fd12b8d555a55Duncan Sands case ISD::FTRUNC: 151364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = WidenVecRes_Unary(N); 151464a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 15153b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper case ISD::FMA: 15163b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper Res = WidenVecRes_Ternary(N); 15173b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper break; 151887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 151987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 152087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If Res is null, the sub-method took care of registering the result. 152187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (Res.getNode()) 152287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SetWidenedVector(SDValue(N, ResNo), Res); 152387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 152487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 15253b9dfc9bf79f1134336fa704c54e411a24856d1dCraig TopperSDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 15263b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper // Ternary op widening. 1527e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 15283b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 15293b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 15303b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 15313b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 15323b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 15333b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper} 15343b9dfc9bf79f1134336fa704c54e411a24856d1dCraig Topper 153587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 153687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Binary op widening. 1537f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang unsigned Opcode = N->getOpcode(); 1538e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 153923b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1540f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang EVT WidenEltVT = WidenVT.getVectorElementType(); 1541f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang EVT VT = WidenVT; 1542f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang unsigned NumElts = VT.getVectorNumElements(); 1543e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1544549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NumElts = NumElts / 2; 1545549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1546f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1547f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang 1548f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1549f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang // Operation doesn't trap so just widen as normal. 1550f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1551f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1552f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1553549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 1554bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 1555549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // No legal vector version so unroll the vector operation and then widen. 1556549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (NumElts == 1) 1557f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1558bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 1559549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // Since the operation can trap, apply operation on the original vector. 1560549fa267efb00944a418a507d07101bae2f72b51Chris Lattner EVT MaxVT = VT; 1561549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1562549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1563549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1564549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 1565549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1566549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned ConcatEnd = 0; // Current ConcatOps index. 1567549fa267efb00944a418a507d07101bae2f72b51Chris Lattner int Idx = 0; // Current Idx into input vectors. 1568549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 1569e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen // NumElts := greatest legal vector size (at most WidenVT) 1570549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // while (orig. vector has unhandled elements) { 1571549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // take munches of size NumElts from the beginning and add to ConcatOps 1572549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // NumElts := next smaller supported vector size or 1 1573549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // } 1574549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (CurNumElts != 0) { 1575549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (CurNumElts >= NumElts) { 1576549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1577425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1578549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1579425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1580549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1581549fa267efb00944a418a507d07101bae2f72b51Chris Lattner Idx += NumElts; 1582549fa267efb00944a418a507d07101bae2f72b51Chris Lattner CurNumElts -= NumElts; 1583549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 1584549fa267efb00944a418a507d07101bae2f72b51Chris Lattner do { 1585549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NumElts = NumElts / 2; 1586549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1587e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1588549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 1589549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (NumElts == 1) { 1590549fa267efb00944a418a507d07101bae2f72b51Chris Lattner for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1591bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1592425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard InOp1, DAG.getConstant(Idx, 1593425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 1594bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1595425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard InOp2, DAG.getConstant(Idx, 1596425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 1597549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1598549fa267efb00944a418a507d07101bae2f72b51Chris Lattner EOp1, EOp2); 1599f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1600549fa267efb00944a418a507d07101bae2f72b51Chris Lattner CurNumElts = 0; 1601f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1602549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 1603f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang 1604549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // Check to see if we have a single operation with the widen type. 1605549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (ConcatEnd == 1) { 1606549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VT = ConcatOps[0].getValueType(); 1607549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (VT == WidenVT) 1608549fa267efb00944a418a507d07101bae2f72b51Chris Lattner return ConcatOps[0]; 1609549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 1610f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang 1611549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // while (Some element of ConcatOps is not of type MaxVT) { 1612549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // From the end of ConcatOps, collect elements of the same type and put 1613549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // them into an op of the next larger supported type 1614549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // } 1615549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1616549fa267efb00944a418a507d07101bae2f72b51Chris Lattner Idx = ConcatEnd - 1; 1617549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VT = ConcatOps[Idx--].getValueType(); 1618549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1619549fa267efb00944a418a507d07101bae2f72b51Chris Lattner Idx--; 1620549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 1621549fa267efb00944a418a507d07101bae2f72b51Chris Lattner int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1622549fa267efb00944a418a507d07101bae2f72b51Chris Lattner EVT NextVT; 1623549fa267efb00944a418a507d07101bae2f72b51Chris Lattner do { 1624549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NextSize *= 2; 1625549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1626e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen } while (!TLI.isTypeLegal(NextVT)); 1627549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 1628549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (!VT.isVector()) { 1629549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1630549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue VecOp = DAG.getUNDEF(NextVT); 1631549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned NumToInsert = ConcatEnd - Idx - 1; 1632549fa267efb00944a418a507d07101bae2f72b51Chris Lattner for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1633549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1634425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard ConcatOps[OpIdx], DAG.getConstant(i, 1635425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 1636f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1637549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatOps[Idx+1] = VecOp; 1638549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatEnd = Idx + 2; 1639549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } else { 1640549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // Vector type, create a CONCAT_VECTORS of type NextVT 1641549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue undefVec = DAG.getUNDEF(VT); 1642549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1643549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1644549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned RealVals = ConcatEnd - Idx - 1; 1645549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned SubConcatEnd = 0; 1646549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned SubConcatIdx = Idx + 1; 1647549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (SubConcatEnd < RealVals) 1648549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1649549fa267efb00944a418a507d07101bae2f72b51Chris Lattner while (SubConcatEnd < OpsToConcat) 1650549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SubConcatOps[SubConcatEnd++] = undefVec; 1651549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1652549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NextVT, &SubConcatOps[0], 1653549fa267efb00944a418a507d07101bae2f72b51Chris Lattner OpsToConcat); 1654549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatEnd = SubConcatIdx + 1; 1655f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1656549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 16579c4a84b4f38790a74d2e4c95870e27e71a79e326Mon P Wang 1658549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // Check to see if we have a single operation with the widen type. 1659549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (ConcatEnd == 1) { 1660549fa267efb00944a418a507d07101bae2f72b51Chris Lattner VT = ConcatOps[0].getValueType(); 1661549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (VT == WidenVT) 1662549fa267efb00944a418a507d07101bae2f72b51Chris Lattner return ConcatOps[0]; 1663f7ea6c3ee89e605c8d0bb7cdb0ade79706c750e8Mon P Wang } 1664bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 1665549fa267efb00944a418a507d07101bae2f72b51Chris Lattner // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1666549fa267efb00944a418a507d07101bae2f72b51Chris Lattner unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1667549fa267efb00944a418a507d07101bae2f72b51Chris Lattner if (NumOps != ConcatEnd ) { 1668549fa267efb00944a418a507d07101bae2f72b51Chris Lattner SDValue UndefVal = DAG.getUNDEF(MaxVT); 1669549fa267efb00944a418a507d07101bae2f72b51Chris Lattner for (unsigned j = ConcatEnd; j < NumOps; ++j) 1670549fa267efb00944a418a507d07101bae2f72b51Chris Lattner ConcatOps[j] = UndefVal; 1671549fa267efb00944a418a507d07101bae2f72b51Chris Lattner } 1672549fa267efb00944a418a507d07101bae2f72b51Chris Lattner return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], NumOps); 167387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 167487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 167587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 167687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(0); 1677e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(N); 1678ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 167923b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 168087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 168187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1682e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 1683e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InEltVT = InVT.getVectorElementType(); 168423b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 168587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 168687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Opcode = N->getOpcode(); 168787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned InVTNumElts = InVT.getVectorNumElements(); 168887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 168996e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 169087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(N->getOperand(0)); 169187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVT = InOp.getValueType(); 169287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVTNumElts = InVT.getVectorNumElements(); 1693593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner if (InVTNumElts == WidenNumElts) { 1694593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner if (N->getNumOperands() == 1) 1695593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InOp); 1696593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1697593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner } 169887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 169987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1700e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(InWidenVT)) { 170187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Because the result and the input are different vector types, widening 170287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the result could create a legal type but widening the input might make 170387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it an illegal type that might lead to repeatedly splitting the input 170487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // and then widening it. To avoid this, we widen the input only if 170587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it results in a legal type. 170687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenNumElts % InVTNumElts == 0) { 170787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Widen the input and call convert on the widened input vector. 170887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumConcat = WidenNumElts/InVTNumElts; 170987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumConcat); 171087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[0] = InOp; 1711e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(InVT); 171287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i = 1; i != NumConcat; ++i) 171387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 1714593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, 1715593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner &Ops[0], NumConcat); 1716593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner if (N->getNumOperands() == 1) 1717593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InVec); 1718593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 171987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 172087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 172187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVTNumElts % WidenNumElts == 0) { 1722593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1723425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard InOp, DAG.getConstant(0, 1724425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 172587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Extract the input and convert the shorten input vector. 1726593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner if (N->getNumOperands() == 1) 1727593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InVal); 1728593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 172987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 173087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 173187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 173287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Otherwise unroll into some nasty scalar code and rebuild the vector. 173387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 1734e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = WidenVT.getVectorElementType(); 173587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 173687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned i; 1737593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner for (i=0; i < MinElts; ++i) { 1738593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1739425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(i, TLI.getVectorIdxTy())); 1740593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner if (N->getNumOperands() == 1) 1741593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1742593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner else 1743593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1744593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner } 174587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1746e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(EltVT); 174787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (; i < WidenNumElts; ++i) 174887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 174987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1750593051b4e2c5fb88fa4acbe3bec92581bef554c0Chris Lattner return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, &Ops[0], WidenNumElts); 175187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 175287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1753956b3490349019163d1d12bb42b82f7e1fcc20eeDan GohmanSDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1754956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1755956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman SDValue InOp = GetWidenedVector(N->getOperand(0)); 1756956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman SDValue ShOp = N->getOperand(1); 1757ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 1758956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman} 1759956b3490349019163d1d12bb42b82f7e1fcc20eeDan Gohman 176087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 176123b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 176287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 176387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue ShOp = N->getOperand(1); 176487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1765e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT ShVT = ShOp.getValueType(); 176696e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 176787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ShOp = GetWidenedVector(ShOp); 176887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ShVT = ShOp.getValueType(); 176987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 1770adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1771adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng ShVT.getVectorElementType(), 177287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang WidenVT.getVectorNumElements()); 1773ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands if (ShVT != ShWidenVT) 177487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ShOp = ModifyToType(ShOp, ShWidenVT); 1775ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 1776ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 177787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 177887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 177987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 178087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Unary op widening. 178123b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 178287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 1783ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 178487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 178587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1786d1996360399ad6dbe75ee185b661b16c83146373Dan GohmanSDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1787d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1788d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1789d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman cast<VTSDNode>(N->getOperand(1))->getVT() 1790d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman .getVectorElementType(), 1791d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman WidenVT.getVectorNumElements()); 1792d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1793ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), 1794d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1795d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman} 1796d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman 17974c19e12d28749c717d3b384962c9ec92796af1c9Duncan SandsSDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 17984c19e12d28749c717d3b384962c9ec92796af1c9Duncan Sands SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 179962bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman return GetWidenedVector(WidenVec); 180062bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman} 180162bb16cfd10dd271eab6c31d982bca4d79138602Eli Friedman 1802bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley PeckSDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 180387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(0); 1804e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 1805e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 180623b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1807e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 180887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 180987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang switch (getTypeAction(InVT)) { 181096e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeLegal: 181187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang break; 181296e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypePromoteInteger: 1813316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem // If the incoming type is a vector that is being promoted, then 1814316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem // we know that the elements are arranged differently and that we 1815316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem // must perform the conversion using a stack slot. 1816316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem if (InVT.isVector()) 1817316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem break; 1818316477dd543b5ae30b832ed4c7708f7aaa51747cNadav Rotem 181987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If the InOp is promoted to the same size, convert it. Otherwise, 182087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // fall out of the switch and widen the promoted input. 182187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetPromotedInteger(InOp); 182287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVT = InOp.getValueType(); 182387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenVT.bitsEq(InVT)) 1824bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 182587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang break; 182696e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeSoftenFloat: 182796e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeExpandInteger: 182896e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeExpandFloat: 182996e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeScalarizeVector: 183096e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeSplitVector: 183187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang break; 183296e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem case TargetLowering::TypeWidenVector: 183387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If the InOp is widened to the same size, convert it. Otherwise, fall 183487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // out of the switch and widen the widened input. 183587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 183687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVT = InOp.getValueType(); 183787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenVT.bitsEq(InVT)) 183887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // The input widens to the same size. Convert to the widen value. 1839bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 184087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang break; 184187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 184287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 184387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenSize = WidenVT.getSizeInBits(); 184487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned InSize = InVT.getSizeInBits(); 184561b20efd9aae99101acf6fd480dee017b702f68bDale Johannesen // x86mmx is not an acceptable vector element type, so don't try. 184661b20efd9aae99101acf6fd480dee017b702f68bDale Johannesen if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 184787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Determine new input vector type. The new input vector type will use 184887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the same element type (if its a vector) or use the input type as a 184987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // vector. It is the same size as the type to widen to. 1850e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT NewInVT; 185187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NewNumElts = WidenSize / InSize; 185287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVT.isVector()) { 1853e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InEltVT = InVT.getVectorElementType(); 1854549fa267efb00944a418a507d07101bae2f72b51Chris Lattner NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1855549fa267efb00944a418a507d07101bae2f72b51Chris Lattner WidenSize / InEltVT.getSizeInBits()); 185687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } else { 185723b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 185887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 185987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1860e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(NewInVT)) { 186187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Because the result and the input are different vector types, widening 186287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the result could create a legal type but widening the input might make 186387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it an illegal type that might lead to repeatedly splitting the input 186487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // and then widening it. To avoid this, we widen the input only if 186587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it results in a legal type. 186687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NewNumElts); 1867e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(InVT); 186887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[0] = InOp; 186987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i = 1; i < NewNumElts; ++i) 187087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 187187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 187287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue NewVec; 187387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVT.isVector()) 1874644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1875644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen NewInVT, &Ops[0], NewNumElts); 187687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang else 1877a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1878a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng NewInVT, &Ops[0], NewNumElts); 1879bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 188087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 188187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 188287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1883aad3460086a1b29c55f7490c6d8743ea4e53f07dEli Friedman return CreateStackStoreLoad(InOp, WidenVT); 188487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 188587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 188687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1887e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 188887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Build a vector with undefined for the new nodes. 1889e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 1890b9c8c40acbe120ebf44d2a81382ceee100bcd331Hal Finkel 1891b9c8c40acbe120ebf44d2a81382ceee100bcd331Hal Finkel // Integer BUILD_VECTOR operands may be larger than the node's vector element 1892b9c8c40acbe120ebf44d2a81382ceee100bcd331Hal Finkel // type. The UNDEFs need to have the same type as the existing operands. 1893b9c8c40acbe120ebf44d2a81382ceee100bcd331Hal Finkel EVT EltVT = N->getOperand(0).getValueType(); 189487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumElts = VT.getVectorNumElements(); 189587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 189623b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 189787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 189887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 189987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1900520b6e3fa5c9c01ed66e70c925e78d518504a983Benjamin Kramer assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 1901520b6e3fa5c9c01ed66e70c925e78d518504a983Benjamin Kramer NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 190287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1903a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 190487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 190587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 190687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1907e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = N->getOperand(0).getValueType(); 190823b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1909e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 191087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 19110381c21d2ddc182aebfef25c6500d781ddb428feEli Friedman unsigned NumInElts = InVT.getVectorNumElements(); 191287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumOperands = N->getNumOperands(); 1913ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 191487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang bool InputWidened = false; // Indicates we need to widen the input. 191596e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 191687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 191787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Add undef vectors to widen to correct length. 1918ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands unsigned NumConcat = WidenVT.getVectorNumElements() / 191987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVT.getVectorNumElements(); 1920e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(InVT); 192187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumConcat); 192287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i=0; i < NumOperands; ++i) 192387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = N->getOperand(i); 192487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i = NumOperands; i != NumConcat; ++i) 192587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 1926644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 192787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 192887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } else { 192987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InputWidened = true; 193023b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 193187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // The inputs and the result are widen to the same value. 193287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned i; 193387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (i=1; i < NumOperands; ++i) 193487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (N->getOperand(i).getOpcode() != ISD::UNDEF) 193587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang break; 193687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 19370381c21d2ddc182aebfef25c6500d781ddb428feEli Friedman if (i == NumOperands) 193887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Everything but the first operand is an UNDEF so just return the 193987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // widened first operand. 194087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return GetWidenedVector(N->getOperand(0)); 194187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 194287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (NumOperands == 2) { 194387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Replace concat of two operands with a shuffle. 19440381c21d2ddc182aebfef25c6500d781ddb428feEli Friedman SmallVector<int, 16> MaskOps(WidenNumElts, -1); 19450381c21d2ddc182aebfef25c6500d781ddb428feEli Friedman for (unsigned i = 0; i < NumInElts; ++i) { 19469008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman MaskOps[i] = i; 19470381c21d2ddc182aebfef25c6500d781ddb428feEli Friedman MaskOps[i + NumInElts] = i + WidenNumElts; 194887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 19498c899ee031481dbece5f111379a274c848cb5902Duncan Sands return DAG.getVectorShuffle(WidenVT, dl, 19509008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman GetWidenedVector(N->getOperand(0)), 19519008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman GetWidenedVector(N->getOperand(1)), 19529008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman &MaskOps[0]); 195387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 195487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 195587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 1956ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 195787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Fall back to use extracts and build vector. 1958e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = WidenVT.getVectorElementType(); 195987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 196087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Idx = 0; 196187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i=0; i < NumOperands; ++i) { 196287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(i); 196387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InputWidened) 196487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 196587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned j=0; j < NumInElts; ++j) 1966f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1967425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(j, TLI.getVectorIdxTy())); 196887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 1969e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(EltVT); 197087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (; Idx < WidenNumElts; ++Idx) 197187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[Idx] = UndefVal; 1972a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 197387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 197487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 197587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1976e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 197787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(0); 197887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue RndOp = N->getOperand(3); 197987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue SatOp = N->getOperand(4); 198087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1981549fa267efb00944a418a507d07101bae2f72b51Chris Lattner EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 198287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 198387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 1984e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 1985e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InEltVT = InVT.getVectorElementType(); 198623b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 198787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 198887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue DTyOp = DAG.getValueType(WidenVT); 198987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue STyOp = DAG.getValueType(InWidenVT); 199087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 199187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 199287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned InVTNumElts = InVT.getVectorNumElements(); 199396e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 199487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 199587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVT = InOp.getValueType(); 199687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InVTNumElts = InVT.getVectorNumElements(); 199787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVTNumElts == WidenNumElts) 1998c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 199987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SatOp, CvtCode); 200087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 200187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2002e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(InWidenVT)) { 200387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Because the result and the input are different vector types, widening 200487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the result could create a legal type but widening the input might make 200587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it an illegal type that might lead to repeatedly splitting the input 200687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // and then widening it. To avoid this, we widen the input only if 200787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // it results in a legal type. 200887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenNumElts % InVTNumElts == 0) { 200987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Widen the input and call convert on the widened input vector. 201087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumConcat = WidenNumElts/InVTNumElts; 201187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumConcat); 201287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[0] = InOp; 2013e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(InVT); 2014549fa267efb00944a418a507d07101bae2f72b51Chris Lattner for (unsigned i = 1; i != NumConcat; ++i) 201587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 2016549fa267efb00944a418a507d07101bae2f72b51Chris Lattner 2017c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 2018c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 201987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SatOp, CvtCode); 202087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 202187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 202287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVTNumElts % WidenNumElts == 0) { 202387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Extract the input and convert the shorten input vector. 2024c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2025425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 2026c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2027f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper SatOp, CvtCode); 202887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 202987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 203087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 203187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Otherwise unroll into some nasty scalar code and rebuild the vector. 203287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 2033e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = WidenVT.getVectorElementType(); 203487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang DTyOp = DAG.getValueType(EltVT); 203587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang STyOp = DAG.getValueType(InEltVT); 203687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 203787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 203887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned i; 203987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (i=0; i < MinElts; ++i) { 2040c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2041425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(i, TLI.getVectorIdxTy())); 2042c460ae90019ddb19d4c07b2cd2fbaecfa7adf67dDale Johannesen Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2043f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper SatOp, CvtCode); 204487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 204587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2046e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(EltVT); 204787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (; i < WidenNumElts; ++i) 204887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 204987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2050a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 205187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 205287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 205387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2054e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 205523b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 205687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 205787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(0); 205887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue Idx = N->getOperand(1); 2059e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 206087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 206196e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 206287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 206387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2064e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 206587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 20666736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson // Check if we can just return the input vector after widening. 20676736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 20686736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson if (IdxVal == 0 && InVT == WidenVT) 20696736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson return InOp; 20706736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson 20716736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson // Check if we can extract from the vector. 20726736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson unsigned InNumElts = InVT.getVectorNumElements(); 20736736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 20746736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 207587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 207687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // We could try widening the input to the right length but for now, extract 207787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the original elements, fill the rest with undefs and build a vector. 207887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 2079e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VT.getVectorElementType(); 208087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumElts = VT.getVectorNumElements(); 208187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned i; 20826736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson for (i=0; i < NumElts; ++i) 20836736e19f4c9af1181cedca54ba6e3a1420454928Bob Wilson Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2084425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy())); 208587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2086e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(EltVT); 208787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (; i < WidenNumElts; ++i) 208887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 2089a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 209087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 209187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 209287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 209387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 2094ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 2095644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen InOp.getValueType(), InOp, 209687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang N->getOperand(1), N->getOperand(2)); 209787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 209887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 209987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 210087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang LoadSDNode *LD = cast<LoadSDNode>(N); 210187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ISD::LoadExtType ExtType = LD->getExtensionType(); 210287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 210387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue Result; 210487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2105eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (ExtType != ISD::NON_EXTLOAD) 2106eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2107eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang else 2108eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Result = GenWidenVectorLoads(LdChain, LD); 2109eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2110eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // If we generate a single load, we can use that for the chain. Otherwise, 2111eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // build a factor node to remember the multiple loads are independent and 2112eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // chain to that. 2113eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue NewChain; 2114eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (LdChain.size() == 1) 2115eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang NewChain = LdChain[0]; 2116eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang else 2117ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, 2118eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang &LdChain[0], LdChain.size()); 211987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 212087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Modified the chain - switch anything that used the old chain to use 212187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the new one. 2122607a0508ba5732716809b926a271a9152bdcec12Dan Gohman ReplaceValueWith(SDValue(N, 1), NewChain); 212387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 212487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return Result; 212587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 212687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 212787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 212823b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2129ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 2130644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen WidenVT, N->getOperand(0)); 213187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 213287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 213387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 213423b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 213587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 213687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 213787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue Cond1 = N->getOperand(0); 2138e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT CondVT = Cond1.getValueType(); 213987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (CondVT.isVector()) { 2140e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT CondEltVT = CondVT.getVectorElementType(); 2141adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2142adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng CondEltVT, WidenNumElts); 214396e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 214487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Cond1 = GetWidenedVector(Cond1); 214587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 214687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (Cond1.getValueType() != CondWidenVT) 2147f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper Cond1 = ModifyToType(Cond1, CondWidenVT); 214887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 214987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 215087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 215187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 215287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2153ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(N->getOpcode(), SDLoc(N), 2154644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen WidenVT, Cond1, InOp1, InOp2); 215587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 215687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 215787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 215887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2159ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2160ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2161644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen InOp1.getValueType(), N->getOperand(0), 216287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 216387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 216487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 21656fb474bd3c3e291973ae8e087787090cf2be73f8Mon P WangSDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 216628b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() == 216728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 216828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Scalar/Vector type mismatch"); 216928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 217028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands 21716fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 21726fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 21736fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2174ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, 21756fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang InOp1, InOp2, N->getOperand(2)); 21766fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang} 21776fb474bd3c3e291973ae8e087787090cf2be73f8Mon P Wang 217887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 217923b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2180e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen return DAG.getUNDEF(WidenVT); 218187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 218287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 21835a5ca1519e04310f585197c20e7ae584b7f2d11fNate BegemanSDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2184e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 2185e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 218687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 218723b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 21885a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman unsigned NumElts = VT.getVectorNumElements(); 218987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 219087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 219187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 219287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 219387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 219487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Adjust mask based on new input vector length. 21959008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman SmallVector<int, 16> NewMask; 21965a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman for (unsigned i = 0; i != NumElts; ++i) { 21975a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman int Idx = N->getMaskElt(i); 21985a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman if (Idx < (int)NumElts) 21999008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman NewMask.push_back(Idx); 22009008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman else 22019008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman NewMask.push_back(Idx - NumElts + WidenNumElts); 220287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 22035a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman for (unsigned i = NumElts; i != WidenNumElts; ++i) 22049008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman NewMask.push_back(-1); 22059008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 220687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 220787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 220887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 220928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands assert(N->getValueType(0).isVector() && 221028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands N->getOperand(0).getValueType().isVector() && 221128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "Operands must be vectors"); 221223b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 221387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 221487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 221587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp1 = N->getOperand(0); 2216e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp1.getValueType(); 221787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang assert(InVT.isVector() && "can not widen non vector type"); 2218adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2219adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng InVT.getVectorElementType(), WidenNumElts); 222087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp1 = GetWidenedVector(InOp1); 222187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 222287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 222387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Assume that the input and output will be widen appropriately. If not, 222487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // we will have to unroll it at some point. 222587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang assert(InOp1.getValueType() == WidenInVT && 222687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp2.getValueType() == WidenInVT && 222787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang "Input not widened to expected type!"); 22281f6a329f79b3568d379142f921f59c4143ddaa14Duncan Sands (void)WidenInVT; 2229ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::SETCC, SDLoc(N), 2230644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen WidenVT, InOp1, InOp2, N->getOperand(2)); 223187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 223287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 223387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 223487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 223587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang// Widen Vector Operand 223687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 22379d796db3e746c31dbdb605510c53b3da98d71b38Michael Liaobool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 22389d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 22394437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 22407419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"); 224187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue Res = SDValue(); 224287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 22439d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao // See if the target wants to custom widen this node. 22449d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 22459d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao return false; 22469d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao 224787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang switch (N->getOpcode()) { 224887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang default: 224987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang#ifndef NDEBUG 22509d796db3e746c31dbdb605510c53b3da98d71b38Michael Liao dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 22514437ae213d5435390f0750213b53ec807c047f22Chris Lattner N->dump(&DAG); 22527419b1806a1c3abdc23b62de76fae737b763fb33David Greene dbgs() << "\n"; 225387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang#endif 2254c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable("Do not know how to widen this operator's operand!"); 225587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2256bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 225787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 22583b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P Wang case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 225987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 226087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::STORE: Res = WidenVecOp_STORE(N); break; 22614bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2262ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 22639c45251e1165a9ed8c351468ebb01b3859ea1df3Duncan Sands case ISD::FP_EXTEND: 226487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FP_TO_SINT: 226587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::FP_TO_UINT: 226687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::SINT_TO_FP: 226764a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::UINT_TO_FP: 226887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang case ISD::TRUNCATE: 226964a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::SIGN_EXTEND: 227064a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ZERO_EXTEND: 227164a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner case ISD::ANY_EXTEND: 227264a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner Res = WidenVecOp_Convert(N); 227364a3fa23f8bd02d5b5f44faec6e9cbc26a417c44Chris Lattner break; 227487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 227587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 227687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If Res is null, the sub-method took care of registering the result. 227787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (!Res.getNode()) return false; 227887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 227987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If the result is N, the sub-method updated N in place. Tell the legalizer 228087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // core about this. 228187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (Res.getNode() == N) 228287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return true; 228387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 228487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 228587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 228687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang "Invalid operand expansion"); 228787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 228887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang ReplaceValueWith(SDValue(N, 0), Res); 228987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return false; 229087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 229187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 229287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 229387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Since the result is legal and the input is illegal, it is unlikely 229487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // that we can fix the input to a legal type so unroll the convert 229587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // into some scalar code and create a nasty build vector. 2296e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 2297e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VT.getVectorElementType(); 2298e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 229987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumElts = VT.getVectorNumElements(); 230087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(0); 230196e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 230287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 2303e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 2304e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InEltVT = InVT.getVectorElementType(); 230587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 230687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Opcode = N->getOpcode(); 230787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumElts); 230887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i=0; i < NumElts; ++i) 2309644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2310644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2311425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(i, TLI.getVectorIdxTy()))); 231287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2313a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 231487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 231587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2316bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley PeckSDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2317e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 2318fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 2319e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InWidenVT = InOp.getValueType(); 2320e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 2321fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang 2322fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang // Check if we can convert between two legal vector types and extract. 2323fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang unsigned InWidenSize = InWidenVT.getSizeInBits(); 2324fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang unsigned Size = VT.getSizeInBits(); 232561b20efd9aae99101acf6fd480dee017b702f68bDale Johannesen // x86mmx is not an acceptable vector element type, so don't try. 232661b20efd9aae99101acf6fd480dee017b702f68bDale Johannesen if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2327fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang unsigned NewNumElts = InWidenSize / Size; 232823b9b19b1a5a00faa9fce0788155c7dbfd00bfb1Owen Anderson EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2329e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(NewVT)) { 2330bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2331644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2332425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 2333fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang } 2334fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang } 2335fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang 2336aad3460086a1b29c55f7490c6d8743ea4e53f07dEli Friedman return CreateStackStoreLoad(InOp, VT); 2337fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang} 2338fa9c5eac33bef67e2faef166c902e3bb16efa30fMon P Wang 233987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 234087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // If the input vector is not legal, it is likely that we will not find a 234187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // legal vector of the same size. Replace the concatenate vector with a 234287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // nasty build vector. 2343e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(0); 2344e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VT.getVectorElementType(); 2345e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 234687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumElts = VT.getVectorNumElements(); 234787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumElts); 234887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2349e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = N->getOperand(0).getValueType(); 235087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumInElts = InVT.getVectorNumElements(); 235187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 235287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Idx = 0; 235387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumOperands = N->getNumOperands(); 235487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i=0; i < NumOperands; ++i) { 235587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = N->getOperand(i); 235696e0c5477c41b316263e894bbb5821c7cdeb25efNadav Rotem if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 235787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang InOp = GetWidenedVector(InOp); 235887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned j=0; j < NumInElts; ++j) 2359644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2360425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(j, TLI.getVectorIdxTy())); 236187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 2362a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 236387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 236487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 23653b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P WangSDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 23663b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 2367ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 23683b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P Wang N->getValueType(0), InOp, N->getOperand(1)); 23693b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P Wang} 23703b7ee20d58f526c3904dc6e068c55e9e07b54c90Mon P Wang 237187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 237287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SDValue InOp = GetWidenedVector(N->getOperand(0)); 2373ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2374c680ac90032bf455b2bba77de538fccea08eb267Eli Friedman N->getValueType(0), InOp, N->getOperand(1)); 237587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 237687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 237787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P WangSDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 237887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // We have to widen the value but we want only to store the original 237987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // vector type. 238087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang StoreSDNode *ST = cast<StoreSDNode>(N); 238187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 238287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> StChain; 2383eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (ST->isTruncatingStore()) 2384eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang GenWidenVectorTruncStores(StChain, ST); 2385eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang else 2386eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang GenWidenVectorStores(StChain, ST); 2387eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 238887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (StChain.size() == 1) 238987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return StChain[0]; 2390ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands else 2391ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::TokenFactor, SDLoc(ST), 2392825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson MVT::Other,&StChain[0],StChain.size()); 239387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 239487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 23954bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav RotemSDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2396ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2397ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2398e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(N); 2399ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem 2400ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem // WARNING: In this code we widen the compare instruction with garbage. 2401ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem // This garbage may contain denormal floats which may be slow. Is this a real 2402ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem // concern ? Should we zero the unused lanes if this is a float compare ? 2403ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem 24044bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem // Get a new SETCC node to compare the newly widened operands. 24054bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem // Only some of the compared elements are legal. 2406225ed7069caae9ece32d8bd3d15c6e41e21cc04bMatt Arsenault EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType()); 2407ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 24084bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem SVT, InOp0, InOp1, N->getOperand(2)); 24094bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem 24104bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem // Extract the needed results from the result vector. 24114bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 24124bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem SVT.getVectorElementType(), 24134bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem N->getValueType(0).getVectorNumElements()); 24144bd222ae26d0411d5c67fd0ab5c043422b5f201bNadav Rotem SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2415425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard ResVT, WideSETCC, DAG.getConstant(0, 2416425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard TLI.getVectorIdxTy())); 2417ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem 2418f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper return PromoteTargetBoolean(CC, N->getValueType(0)); 2419ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem} 2420ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem 2421ca58c722674ed7b564f68d1d9697cd01504edef2Nadav Rotem 242287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 242387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang// Vector Widening Utilities 242487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang//===----------------------------------------------------------------------===// 242587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2426eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// Utility function to find the type to chop up a widen vector for load/store 2427eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// TLI: Target lowering used to determine legal types. 2428eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// Width: Width left need to load/store. 2429eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// WidenVT: The widen vector type to load to/store from 2430eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// Align: If 0, don't allow use of a wider type 2431eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// WidenEx: If Align is not 0, the amount additional we can load/store from. 2432eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2433eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wangstatic EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2434eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Width, EVT WidenVT, 2435eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Align = 0, unsigned WidenEx = 0) { 2436eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT WidenEltVT = WidenVT.getVectorElementType(); 2437eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned WidenWidth = WidenVT.getSizeInBits(); 2438eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2439eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned AlignInBits = Align*8; 2440eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2441eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // If we have one element to load/store, return it. 2442eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT RetVT = WidenEltVT; 2443eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (Width == WidenEltWidth) 2444eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang return RetVT; 2445eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2446bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck // See if there is larger legal integer than the element type to load/store 2447eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned VT; 2448eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2449eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2450eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT MemVT((MVT::SimpleValueType) VT); 2451eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned MemVTWidth = MemVT.getSizeInBits(); 2452eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (MemVT.getSizeInBits() <= WidenEltWidth) 2453eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang break; 2454e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 245500404bfaef2cd9516191fd35cc08a94a536b1048Nadav Rotem isPowerOf2_32(WidenWidth / MemVTWidth) && 2456eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang (MemVTWidth <= Width || 2457eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2458eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang RetVT = MemVT; 2459eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang break; 2460eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2461eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 246287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2463eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // See if there is a larger vector type to load/store that has the same vector 2464eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // element type and is evenly divisible with the WidenVT. 2465eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2466eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2467eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT MemVT = (MVT::SimpleValueType) VT; 2468eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned MemVTWidth = MemVT.getSizeInBits(); 2469e93d99cf0742eebab859022e4cfdcf03cb9d5dfaDale Johannesen if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2470eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang (WidenWidth % MemVTWidth) == 0 && 247100404bfaef2cd9516191fd35cc08a94a536b1048Nadav Rotem isPowerOf2_32(WidenWidth / MemVTWidth) && 2472eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang (MemVTWidth <= Width || 2473eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2474eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2475eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang return MemVT; 2476eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2477eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2478eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2479eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang return RetVT; 2480eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang} 2481eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2482eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// Builds a vector type from scalar loads 2483eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// VecTy: Resulting Vector type 2484eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// LDOps: Load operators to build a vector type 2485eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang// [Start,End) the list of loads to use. 2486eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wangstatic SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2487a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Topper SmallVectorImpl<SDValue> &LdOps, 2488eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Start, unsigned End) { 2489425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2490e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(LdOps[Start]); 2491eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT LdTy = LdOps[Start].getValueType(); 2492eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Width = VecTy.getSizeInBits(); 2493eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NumElts = Width / LdTy.getSizeInBits(); 2494eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2495eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2496eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Idx = 1; 2497eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2498eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2499eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (unsigned i = Start + 1; i != End; ++i) { 2500eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewLdTy = LdOps[i].getValueType(); 2501eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (NewLdTy != LdTy) { 2502eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang NumElts = Width / NewLdTy.getSizeInBits(); 2503eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2504bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2505eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Readjust position and vector position based on new load type 2506eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2507eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdTy = NewLdTy; 2508eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2509eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2510425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 251187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 2512bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 251387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 251487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2515a0ec3f9b7b826b9b40b80199923b664bad808cceCraig TopperSDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 2516ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner LoadSDNode *LD) { 251787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // The strategy assumes that we can efficiently load powers of two widths. 2518eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // The routines chops the vector into the largest vector loads with the same 2519eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // element type or scalar loads and then recombines it to the widen vector 2520eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // type. 2521adf979900c84d00e1fe0872a68d2819c654b6f29Evan Cheng EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2522eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned WidenWidth = WidenVT.getSizeInBits(); 2523eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT LdVT = LD->getMemoryVT(); 2524e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(LD); 2525eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(LdVT.isVector() && WidenVT.isVector()); 2526eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2527eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2528eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Load information 2529eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue Chain = LD->getChain(); 2530eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue BasePtr = LD->getBasePtr(); 2531eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Align = LD->getAlignment(); 2532eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang bool isVolatile = LD->isVolatile(); 25331e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNonTemporal = LD->isNonTemporal(); 2534d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper bool isInvariant = LD->isInvariant(); 253587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2536eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang int LdWidth = LdVT.getSizeInBits(); 2537eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang int WidthDiff = WidenWidth - LdWidth; // Difference 2538eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 253987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 254087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Find the vector type that can load from. 2541eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2542eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang int NewVTWidth = NewVT.getSizeInBits(); 2543ecf42c4720aba6ee315d0166045c54187ac2de4dChris Lattner SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2544d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper isVolatile, isNonTemporal, isInvariant, Align); 254587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang LdChain.push_back(LdOp.getValue(1)); 254687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 254787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Check if we can load the element with one instruction 2548eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (LdWidth <= NewVTWidth) { 25498306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (!NewVT.isVector()) { 2550c1e5d491a0bd99a71c87f6a69bafbecd2f948c82Mon P Wang unsigned NumElts = WidenWidth / NewVTWidth; 2551eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2552eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2553bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2554eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 25558306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (NewVT == WidenVT) 25568306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return LdOp; 25578306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner 25588306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner assert(WidenWidth % NewVTWidth == 0); 25598306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner unsigned NumConcat = WidenWidth / NewVTWidth; 25608306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SmallVector<SDValue, 16> ConcatOps(NumConcat); 25618306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SDValue UndefVal = DAG.getUNDEF(NewVT); 25628306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[0] = LdOp; 25638306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (unsigned i = 1; i != NumConcat; ++i) 25648306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[i] = UndefVal; 25658306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], 25668306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner NumConcat); 256787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 256887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2569eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Load vector by using multiple loads from largest vector to scalar 2570eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SmallVector<SDValue, 16> LdOps; 2571eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdOps.push_back(LdOp); 2572eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2573eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdWidth -= NewVTWidth; 257487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Offset = 0; 2575ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 257687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang while (LdWidth > 0) { 2577aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned Increment = NewVTWidth / 8; 257887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Offset += Increment; 2579644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 258087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang DAG.getIntPtrConstant(Increment)); 258187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2582c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem SDValue L; 2583eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (LdWidth < NewVTWidth) { 2584eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Our current type we are using is too large, find a better size 2585eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2586eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang NewVTWidth = NewVT.getSizeInBits(); 2587c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2588f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2589f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2590c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem LdChain.push_back(L.getValue(1)); 2591c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem if (L->getValueType(0).isVector()) { 2592c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem SmallVector<SDValue, 16> Loads; 2593c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem Loads.push_back(L); 2594c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem unsigned size = L->getValueSizeInBits(0); 2595c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem while (size < LdOp->getValueSizeInBits(0)) { 2596c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2597c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem size += L->getValueSizeInBits(0); 2598c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem } 2599c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), 2600c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem &Loads[0], Loads.size()); 2601c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem } 2602c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem } else { 2603c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2604c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2605c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2606c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem LdChain.push_back(L.getValue(1)); 2607c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem } 2608c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem 2609c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem LdOps.push_back(L); 2610c8d12eee12bbd0dca3def72d52e410eaf4e61b2dNadav Rotem 2611ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 2612eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdWidth -= NewVTWidth; 261387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 261487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2615eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Build the vector from the loads operations 2616eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned End = LdOps.size(); 26178306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (!LdOps[0].getValueType().isVector()) 26188306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // All the loads are scalar loads. 26198306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2620bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 26218306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // If the load contains vectors, build the vector using concat vector. 26228306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // All of the vectors used to loads are power of 2 and the scalars load 26238306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // can be combined to make a power of 2 vector. 26248306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SmallVector<SDValue, 16> ConcatOps(End); 26258306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner int i = End - 1; 26268306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner int Idx = End; 26278306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner EVT LdTy = LdOps[i].getValueType(); 26288306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // First combine the scalar loads to a vector 26298306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (!LdTy.isVector()) { 2630eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (--i; i >= 0; --i) { 26318306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner LdTy = LdOps[i].getValueType(); 26328306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (LdTy.isVector()) 26338306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner break; 2634eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 26358306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 26368306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 26378306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[--Idx] = LdOps[i]; 26388306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (--i; i >= 0; --i) { 26398306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner EVT NewLdTy = LdOps[i].getValueType(); 26408306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (NewLdTy != LdTy) { 26418306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // Create a larger vector 26428306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 26438306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner &ConcatOps[Idx], End - Idx); 26448306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner Idx = End - 1; 26458306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner LdTy = NewLdTy; 26468306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 26478306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner ConcatOps[--Idx] = LdOps[i]; 26488306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 264987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 26508306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 26518306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 26528306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner &ConcatOps[Idx], End - Idx); 26538306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner 26548306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner // We need to fill the rest with undefs to build the vector 26558306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 26568306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SmallVector<SDValue, 16> WidenOps(NumOps); 26578306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner SDValue UndefVal = DAG.getUNDEF(LdTy); 26588306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner { 26598306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner unsigned i = 0; 26608306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (; i != End-Idx; ++i) 26618306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner WidenOps[i] = ConcatOps[Idx+i]; 26628306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner for (; i != NumOps; ++i) 26638306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner WidenOps[i] = UndefVal; 26648306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner } 26658306968c147d5861d8a53fba86ac0fbf5c050b84Chris Lattner return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps); 2666eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang} 2667eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2668eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P WangSDValue 2669a0ec3f9b7b826b9b40b80199923b664bad808cceCraig TopperDAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 2670a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Topper LoadSDNode *LD, 2671eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang ISD::LoadExtType ExtType) { 2672eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // For extension loads, it may not be more efficient to chop up the vector 2673eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // and then extended it. Instead, we unroll the load and build a new vector. 2674eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2675eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT LdVT = LD->getMemoryVT(); 2676e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(LD); 2677eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(LdVT.isVector() && WidenVT.isVector()); 267887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2679eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Load information 2680eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue Chain = LD->getChain(); 2681eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue BasePtr = LD->getBasePtr(); 2682eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Align = LD->getAlignment(); 2683eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang bool isVolatile = LD->isVolatile(); 26841e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNonTemporal = LD->isNonTemporal(); 2685eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2686eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT EltVT = WidenVT.getVectorElementType(); 2687eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT LdEltVT = LdVT.getVectorElementType(); 2688eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NumElts = LdVT.getVectorNumElements(); 2689eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2690eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Load each element and widen 2691eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2692eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 2693aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned Increment = LdEltVT.getSizeInBits() / 8; 2694a901129169194881a78b7fd8953e09f55b846d10Stuart Hastings Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 26953d6ccfba314ed38e4506dae2781a060e9a3e07acChris Lattner LD->getPointerInfo(), 26961e559443a17d1b335f697551c6263ba60d5dd827David Greene LdEltVT, isVolatile, isNonTemporal, Align); 2697eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdChain.push_back(Ops[0].getValue(1)); 2698eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned i = 0, Offset = Increment; 2699eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (i=1; i < NumElts; ++i, Offset += Increment) { 2700eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2701eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang BasePtr, DAG.getIntPtrConstant(Offset)); 2702a901129169194881a78b7fd8953e09f55b846d10Stuart Hastings Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 27033d6ccfba314ed38e4506dae2781a060e9a3e07acChris Lattner LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 27043d6ccfba314ed38e4506dae2781a060e9a3e07acChris Lattner isVolatile, isNonTemporal, Align); 2705eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang LdChain.push_back(Ops[i].getValue(1)); 270687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 2707ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 2708eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Fill the rest with undefs 2709eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue UndefVal = DAG.getUNDEF(EltVT); 2710eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (; i != WidenNumElts; ++i) 2711eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Ops[i] = UndefVal; 2712ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 2713eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 2714eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang} 2715ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 2716ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 2717a0ec3f9b7b826b9b40b80199923b664bad808cceCraig Toppervoid DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 2718eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang StoreSDNode *ST) { 2719eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // The strategy assumes that we can efficiently store powers of two widths. 2720eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // The routines chops the vector into the largest vector stores with the same 2721eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // element type or scalar stores. 2722eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue Chain = ST->getChain(); 2723eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue BasePtr = ST->getBasePtr(); 2724eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Align = ST->getAlignment(); 2725eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang bool isVolatile = ST->isVolatile(); 27261e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNonTemporal = ST->isNonTemporal(); 2727eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue ValOp = GetWidenedVector(ST->getValue()); 2728e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(ST); 2729eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2730eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT StVT = ST->getMemoryVT(); 2731eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned StWidth = StVT.getSizeInBits(); 2732eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT ValVT = ValOp.getValueType(); 2733eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned ValWidth = ValVT.getSizeInBits(); 2734eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT ValEltVT = ValVT.getVectorElementType(); 2735eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2736eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(StVT.getVectorElementType() == ValEltVT); 2737eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2738eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang int Idx = 0; // current index to store 2739eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Offset = 0; // offset from base to store 2740eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang while (StWidth != 0) { 2741eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Find the largest vector type we can store with 2742eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2743eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NewVTWidth = NewVT.getSizeInBits(); 2744aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned Increment = NewVTWidth / 8; 2745eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang if (NewVT.isVector()) { 2746eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NumVTElts = NewVT.getVectorNumElements(); 2747eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang do { 2748eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2749425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx, TLI.getVectorIdxTy())); 2750da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2751da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner ST->getPointerInfo().getWithOffset(Offset), 2752da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner isVolatile, isNonTemporal, 2753eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang MinAlign(Align, Offset))); 2754eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang StWidth -= NewVTWidth; 2755eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Offset += Increment; 2756eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Idx += NumVTElts; 2757eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2758eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang DAG.getIntPtrConstant(Increment)); 2759eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } while (StWidth != 0 && StWidth >= NewVTWidth); 2760eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } else { 2761eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Cast the vector to the scalar type we can store 2762eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NumElts = ValWidth / NewVTWidth; 2763eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2764bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2765eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Readjust index position based on new vector type 2766eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Idx = Idx * ValEltWidth / NewVTWidth; 2767eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang do { 2768eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2769425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2770da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2771da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner ST->getPointerInfo().getWithOffset(Offset), 2772da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner isVolatile, isNonTemporal, 2773da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner MinAlign(Align, Offset))); 2774eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang StWidth -= NewVTWidth; 2775eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Offset += Increment; 2776eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2777eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang DAG.getIntPtrConstant(Increment)); 2778f093393b9a65eae6b04c487784cb8256b15b790eCraig Topper } while (StWidth != 0 && StWidth >= NewVTWidth); 2779eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // Restore index back to be relative to the original widen element type 2780eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang Idx = Idx * NewVTWidth / ValEltWidth; 2781eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2782eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang } 2783eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang} 2784eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2785eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wangvoid 2786a0ec3f9b7b826b9b40b80199923b664bad808cceCraig TopperDAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 2787eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang StoreSDNode *ST) { 2788eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // For extension loads, it may not be more efficient to truncate the vector 2789eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // and then store it. Instead, we extract each element and then store it. 2790eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue Chain = ST->getChain(); 2791eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue BasePtr = ST->getBasePtr(); 2792eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Align = ST->getAlignment(); 2793eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang bool isVolatile = ST->isVolatile(); 27941e559443a17d1b335f697551c6263ba60d5dd827David Greene bool isNonTemporal = ST->isNonTemporal(); 2795eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue ValOp = GetWidenedVector(ST->getValue()); 2796e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(ST); 2797bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck 2798eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT StVT = ST->getMemoryVT(); 2799eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT ValVT = ValOp.getValueType(); 2800eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2801eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // It must be true that we the widen vector type is bigger than where 2802eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // we need to store. 2803eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(StVT.isVector() && ValOp.getValueType().isVector()); 2804eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang assert(StVT.bitsLT(ValOp.getValueType())); 2805eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang 2806eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // For truncating stores, we can not play the tricks of chopping legal 2807eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // vector types and bit cast it to the right type. Instead, we unroll 2808eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang // the store. 2809eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT StEltVT = StVT.getVectorElementType(); 2810eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang EVT ValEltVT = ValVT.getVectorElementType(); 2811aa9d854b334cab2f29ca6d95413a0946b8a38429Dan Gohman unsigned Increment = ValEltVT.getSizeInBits() / 8; 2812eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned NumElts = StVT.getVectorNumElements(); 2813eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2814425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 2815da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 2816da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner ST->getPointerInfo(), StEltVT, 28171e559443a17d1b335f697551c6263ba60d5dd827David Greene isVolatile, isNonTemporal, Align)); 2818eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang unsigned Offset = Increment; 2819eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 2820eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2821eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang BasePtr, DAG.getIntPtrConstant(Offset)); 2822eb38ebf15c326a5bb45ca9da6329cdf19ad6df95Mon P Wang SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2823425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 2824da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 2825da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner ST->getPointerInfo().getWithOffset(Offset), 2826da2d8e1032eb4c2fefb1f647d7877910b9483835Chris Lattner StEltVT, isVolatile, isNonTemporal, 28271e559443a17d1b335f697551c6263ba60d5dd827David Greene MinAlign(Align, Offset))); 282887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 282987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 283087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 283187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang/// Modifies a vector input (widen or narrows) to a vector of NVT. The 283287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang/// input vector must have the same element type as NVT. 2833e50ed30282bb5b4a9ed952580523f2dda16215acOwen AndersonSDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 283487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Note that InOp might have been widened so it might already have 283587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // the right width or it might need be narrowed. 2836e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT InVT = InOp.getValueType(); 283787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 283887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang "input and widen element type must match"); 2839e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(InOp); 284087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 284187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Check if InOp already has the right width. 284287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (InVT == NVT) 284387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang return InOp; 284487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 284587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned InNumElts = InVT.getVectorNumElements(); 2846ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands unsigned WidenNumElts = NVT.getVectorNumElements(); 284787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 284887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned NumConcat = WidenNumElts / InNumElts; 284987c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(NumConcat); 2850e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(InVT); 285187c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[0] = InOp; 285287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (unsigned i = 1; i != NumConcat; ++i) 285387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[i] = UndefVal; 285487c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2855644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 285687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang } 2857ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 285887c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2859644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2860425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(0, TLI.getVectorIdxTy())); 2861ba6d26275f5d0561b9afdc59ebe1d11567fa4fdeDuncan Sands 286287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang // Fall back to extract and build. 286387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang SmallVector<SDValue, 16> Ops(WidenNumElts); 2864e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = NVT.getVectorElementType(); 286587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 286687c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang unsigned Idx; 286787c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for (Idx = 0; Idx < MinNumElts; ++Idx) 2868644f6f875ee6dae47a56692f36ae566c27008e07Dale Johannesen Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2869425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(Idx, TLI.getVectorIdxTy())); 287087c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang 2871e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen SDValue UndefVal = DAG.getUNDEF(EltVT); 287287c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang for ( ; Idx < WidenNumElts; ++Idx) 287387c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang Ops[Idx] = UndefVal; 2874a87008d90b7d894cfca53d407642acfd7be2af3cEvan Cheng return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 287587c8a8f304d1ee72829086ce2c41a8fa3813ba6aMon P Wang} 2876