15c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===-- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ---===// 25c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 35c22c8074404797f1313b1334757254fb5c6487aEli Friedman// The LLVM Compiler Infrastructure 45c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 55c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This file is distributed under the University of Illinois Open Source 65c22c8074404797f1313b1334757254fb5c6487aEli Friedman// License. See LICENSE.TXT for details. 75c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 85c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===----------------------------------------------------------------------===// 95c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 105c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This file implements the SelectionDAG::LegalizeVectors method. 115c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 125c22c8074404797f1313b1334757254fb5c6487aEli Friedman// The vector legalizer looks for vector operations which might need to be 13509150f973ae650a57b79010a3ec36e60e40f41dEli Friedman// scalarized and legalizes them. This is a separate step from Legalize because 14509150f973ae650a57b79010a3ec36e60e40f41dEli Friedman// scalarizing can introduce illegal types. For example, suppose we have an 155c22c8074404797f1313b1334757254fb5c6487aEli Friedman// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition 165c22c8074404797f1313b1334757254fb5c6487aEli Friedman// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the 175c22c8074404797f1313b1334757254fb5c6487aEli Friedman// operation, which introduces nodes with the illegal type i64 which must be 185c22c8074404797f1313b1334757254fb5c6487aEli Friedman// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC; 195c22c8074404797f1313b1334757254fb5c6487aEli Friedman// the operation must be unrolled, which introduces nodes with the illegal 205c22c8074404797f1313b1334757254fb5c6487aEli Friedman// type i8 which must be promoted. 215c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 225c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This does not legalize vector manipulations like ISD::BUILD_VECTOR, 2398ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman// or operations that happen to take a vector which are custom-lowered; 2498ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman// the legalization for such operations never produces nodes 255c22c8074404797f1313b1334757254fb5c6487aEli Friedman// with illegal types, so it's okay to put off legalizing them until 265c22c8074404797f1313b1334757254fb5c6487aEli Friedman// SelectionDAG::Legalize runs. 275c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 285c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===----------------------------------------------------------------------===// 295c22c8074404797f1313b1334757254fb5c6487aEli Friedman 305c22c8074404797f1313b1334757254fb5c6487aEli Friedman#include "llvm/CodeGen/SelectionDAG.h" 315c22c8074404797f1313b1334757254fb5c6487aEli Friedman#include "llvm/Target/TargetLowering.h" 325c22c8074404797f1313b1334757254fb5c6487aEli Friedmanusing namespace llvm; 335c22c8074404797f1313b1334757254fb5c6487aEli Friedman 345c22c8074404797f1313b1334757254fb5c6487aEli Friedmannamespace { 355c22c8074404797f1313b1334757254fb5c6487aEli Friedmanclass VectorLegalizer { 365c22c8074404797f1313b1334757254fb5c6487aEli Friedman SelectionDAG& DAG; 37d858e90f039f5fcdc2fa93035e911a5a9505cc50Dan Gohman const TargetLowering &TLI; 385c22c8074404797f1313b1334757254fb5c6487aEli Friedman bool Changed; // Keep track of whether anything changed 395c22c8074404797f1313b1334757254fb5c6487aEli Friedman 40c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// For nodes that are of legal width, and that have more than one use, this 41c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// map indicates what regularized operand to use. This allows us to avoid 42c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// legalizing the same thing more than once. 43ea387fc3b8cf12c3c6ad218b81eca156e8173bbaPreston Gurd SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes; 445c22c8074404797f1313b1334757254fb5c6487aEli Friedman 45c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Adds a node to the translation cache. 465c22c8074404797f1313b1334757254fb5c6487aEli Friedman void AddLegalizedOperand(SDValue From, SDValue To) { 475c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.insert(std::make_pair(From, To)); 485c22c8074404797f1313b1334757254fb5c6487aEli Friedman // If someone requests legalization of the new node, return itself. 495c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (From != To) 505c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.insert(std::make_pair(To, To)); 515c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 525c22c8074404797f1313b1334757254fb5c6487aEli Friedman 53c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Legalizes the given node. 545c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue LegalizeOp(SDValue Op); 55c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 56c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Assuming the node is legal, "legalize" the results. 575c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue TranslateLegalizeResults(SDValue Op, SDValue Result); 58c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 59c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implements unrolling a VSETCC. 605c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue UnrollVSETCC(SDValue Op); 61c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 62c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement expand-based legalization of vector operations. 63c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 64c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// This is just a high-level routine to dispatch to specific code paths for 65c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// operations to legalize them. 66c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Expand(SDValue Op); 67c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 68c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implements expansion for FNEG; falls back to UnrollVectorOp if 69c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// FSUB isn't legal. 70c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 71c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if 72c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// SINT_TO_FLOAT and SHR on vectors isn't legal. 7306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue ExpandUINT_TO_FLOAT(SDValue Op); 74c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 75c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement expansion for SIGN_EXTEND_INREG using SRL and SRA. 7666de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem SDValue ExpandSEXTINREG(SDValue Op); 77c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 78c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement expansion for ANY_EXTEND_VECTOR_INREG. 79c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 80c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// Shuffles the low lanes of the operand into place and bitcasts to the proper 81c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// type. The contents of the bits in the extended part of each element are 82c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// undef. 83c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op); 84c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 85c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement expansion for SIGN_EXTEND_VECTOR_INREG. 86c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 87c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// Shuffles the low lanes of the operand into place, bitcasts to the proper 88c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// type, then shifts left and arithmetic shifts right to introduce a sign 89c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// extension. 90c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op); 91c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 92c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement expansion for ZERO_EXTEND_VECTOR_INREG. 93c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 94c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// Shuffles the low lanes of the operand into place and blends zeros into 95c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// the remaining lanes, finally bitcasting to the proper type. 96c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op); 97c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 98c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Expand bswap of vectors into a shuffle if legal. 99dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ExpandBSWAP(SDValue Op); 100c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 101c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implement vselect in terms of XOR, AND, OR when blend is not 102c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// supported by the target. 103aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue ExpandVSELECT(SDValue Op); 104e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue ExpandSELECT(SDValue Op); 105e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue ExpandLoad(SDValue Op); 106e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue ExpandStore(SDValue Op); 1075c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue ExpandFNEG(SDValue Op); 108cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SDValue ExpandBITREVERSE(SDValue Op); 109c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 110c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implements vector promotion. 111c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 112c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// This is essentially just bitcasting the operands to a different type and 113c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// bitcasting the result back to the original type. 114c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Promote(SDValue Op); 115c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 116c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implements [SU]INT_TO_FP vector promotion. 117c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 118c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// This is a [zs]ext of the input operand to the next size up. 119c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue PromoteINT_TO_FP(SDValue Op); 120c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 121c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Implements FP_TO_[SU]INT vector promotion of the result type. 122c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// 123c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// It is promoted to the next size up integer type. The result is then 124c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// truncated back to the original type. 125c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue PromoteFP_TO_INT(SDValue Op, bool isSigned); 126c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 127c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinespublic: 128c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief Begin legalizer the vector operations in the DAG. 1295c22c8074404797f1313b1334757254fb5c6487aEli Friedman bool Run(); 1305c22c8074404797f1313b1334757254fb5c6487aEli Friedman VectorLegalizer(SelectionDAG& dag) : 1315c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(false) {} 1325c22c8074404797f1313b1334757254fb5c6487aEli Friedman}; 1335c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1345c22c8074404797f1313b1334757254fb5c6487aEli Friedmanbool VectorLegalizer::Run() { 135d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // Before we start legalizing vector nodes, check if there are any vectors. 136d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem bool HasVectors = false; 137d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 13836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) { 139d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // Check if the values of the nodes contain vectors. We don't need to check 140d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // the operands because we are going to check their values at some point. 141d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem for (SDNode::value_iterator J = I->value_begin(), E = I->value_end(); 142d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem J != E; ++J) 143d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem HasVectors |= J->isVector(); 144d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 145d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // If we found a vector node we can start the legalization. 146d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem if (HasVectors) 147d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem break; 148d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem } 149d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 150d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // If this basic block has no vectors then no need to legalize vectors. 151d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem if (!HasVectors) 152d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem return false; 153d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 1545c22c8074404797f1313b1334757254fb5c6487aEli Friedman // The legalize process is inherently a bottom-up recursive process (users 1555c22c8074404797f1313b1334757254fb5c6487aEli Friedman // legalize their uses before themselves). Given infinite stack space, we 1565c22c8074404797f1313b1334757254fb5c6487aEli Friedman // could just start legalizing on the root and traverse the whole graph. In 1575c22c8074404797f1313b1334757254fb5c6487aEli Friedman // practice however, this causes us to run out of stack space on large basic 1585c22c8074404797f1313b1334757254fb5c6487aEli Friedman // blocks. To avoid this problem, compute an ordering of the nodes where each 1595c22c8074404797f1313b1334757254fb5c6487aEli Friedman // node is only legalized after all of its operands are legalized. 1605c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.AssignTopologicalOrder(); 1615c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 16236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) 163cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar LegalizeOp(SDValue(&*I, 0)); 1645c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1655c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Finally, it's possible the root changed. Get the new root. 1665c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue OldRoot = DAG.getRoot(); 1675c22c8074404797f1313b1334757254fb5c6487aEli Friedman assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 1685c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.setRoot(LegalizedNodes[OldRoot]); 1695c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1705c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.clear(); 1715c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1725c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Remove dead nodes now. 1735c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.RemoveDeadNodes(); 1745c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1755c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Changed; 1765c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 1775c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1785c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) { 1795c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Generic legalization: just pass the operand through. 1805c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned i = 0, e = Op.getNode()->getNumValues(); i != e; ++i) 1815c22c8074404797f1313b1334757254fb5c6487aEli Friedman AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 1825c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Result.getValue(Op.getResNo()); 1835c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 1845c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1855c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::LegalizeOp(SDValue Op) { 1865c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Note that LegalizeOp may be reentered even from single-use nodes, which 1875c22c8074404797f1313b1334757254fb5c6487aEli Friedman // means that we always must cache transformed nodes. 1885c22c8074404797f1313b1334757254fb5c6487aEli Friedman DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1895c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (I != LegalizedNodes.end()) return I->second; 1905c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1915c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDNode* Node = Op.getNode(); 1925c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1935c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Legalize the operands 1945c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 8> Ops; 195cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar for (const SDValue &Op : Node->op_values()) 196cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar Ops.push_back(LegalizeOp(Op)); 1975c22c8074404797f1313b1334757254fb5c6487aEli Friedman 198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0); 1995c22c8074404797f1313b1334757254fb5c6487aEli Friedman 2006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool HasVectorValue = false; 201e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (Op.getOpcode() == ISD::LOAD) { 202e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 203e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ISD::LoadExtType ExtType = LD->getExtensionType(); 20437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) 205ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0), 206ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LD->getMemoryVT())) { 20737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: llvm_unreachable("This action is not supported yet!"); 20837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case TargetLowering::Legal: 209e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TranslateLegalizeResults(Op, Result); 21037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case TargetLowering::Custom: 21137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (SDValue Lowered = TLI.LowerOperation(Result, DAG)) { 212ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Lowered == Result) 213ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return TranslateLegalizeResults(Op, Lowered); 21437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Changed = true; 21537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Lowered->getNumValues() != Op->getNumValues()) { 21637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This expanded to something other than the load. Assume the 21737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // lowering code took care of any chain values, and just handle the 21837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // returned value. 21937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Result.getValue(1).use_empty() && 22037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "There are still live users of the old chain!"); 22137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return LegalizeOp(Lowered); 22237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 223cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return TranslateLegalizeResults(Op, Lowered); 22437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 22537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case TargetLowering::Expand: 22637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Changed = true; 22737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return LegalizeOp(ExpandLoad(Op)); 22837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 229e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } else if (Op.getOpcode() == ISD::STORE) { 230e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 231e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT StVT = ST->getMemoryVT(); 23288ef514cc63c3f22f78eaf4dd295d349b4070819Patrik Hagglund MVT ValVT = ST->getValue().getSimpleValueType(); 233e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (StVT.isVector() && ST->isTruncatingStore()) 234cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar switch (TLI.getTruncStoreAction(ValVT, StVT)) { 2355e25ee8a1fcf8288d00d731b0f7ab7976f33b123Craig Topper default: llvm_unreachable("This action is not supported yet!"); 236e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem case TargetLowering::Legal: 237e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TranslateLegalizeResults(Op, Result); 238ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case TargetLowering::Custom: { 239ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue Lowered = TLI.LowerOperation(Result, DAG); 240ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Changed = Lowered != Result; 241ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return TranslateLegalizeResults(Op, Lowered); 242ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 243e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem case TargetLowering::Expand: 244e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Changed = true; 245e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return LegalizeOp(ExpandStore(Op)); 246e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 247cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar } else if (Op.getOpcode() == ISD::MSCATTER || Op.getOpcode() == ISD::MSTORE) 2486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HasVectorValue = true; 249e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 2505c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end(); 2515c22c8074404797f1313b1334757254fb5c6487aEli Friedman J != E; 2525c22c8074404797f1313b1334757254fb5c6487aEli Friedman ++J) 2535c22c8074404797f1313b1334757254fb5c6487aEli Friedman HasVectorValue |= J->isVector(); 2545c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (!HasVectorValue) 2555c22c8074404797f1313b1334757254fb5c6487aEli Friedman return TranslateLegalizeResults(Op, Result); 2565c22c8074404797f1313b1334757254fb5c6487aEli Friedman 257e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT QueryType; 2585c22c8074404797f1313b1334757254fb5c6487aEli Friedman switch (Op.getOpcode()) { 2595c22c8074404797f1313b1334757254fb5c6487aEli Friedman default: 2605c22c8074404797f1313b1334757254fb5c6487aEli Friedman return TranslateLegalizeResults(Op, Result); 2615c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ADD: 2625c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SUB: 2635c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::MUL: 2645c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SDIV: 2655c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::UDIV: 2665c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SREM: 2675c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::UREM: 268cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::SDIVREM: 269cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::UDIVREM: 2705c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FADD: 2715c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSUB: 2725c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FMUL: 2735c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FDIV: 2745c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FREM: 2755c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::AND: 2765c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::OR: 2775c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::XOR: 2785c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SHL: 2795c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SRA: 2805c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SRL: 2815c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ROTL: 2825c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ROTR: 28336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::BSWAP: 284cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::BITREVERSE: 2855c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::CTLZ: 28663974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTTZ: 28763974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTLZ_ZERO_UNDEF: 28863974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTTZ_ZERO_UNDEF: 2895c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::CTPOP: 2905c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SELECT: 291aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem case ISD::VSELECT: 2925c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SELECT_CC: 29328b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands case ISD::SETCC: 2945c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ZERO_EXTEND: 2955c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ANY_EXTEND: 2965c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::TRUNCATE: 2975c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SIGN_EXTEND: 2985c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FP_TO_SINT: 2995c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FP_TO_UINT: 3005c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FNEG: 3015c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FABS: 30237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::FMINNUM: 30337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::FMAXNUM: 304cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::FMINNAN: 305cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::FMAXNAN: 30666d1fa6f4b443ac9f8bcea5d1f71a73ada733a42Hal Finkel case ISD::FCOPYSIGN: 3075c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSQRT: 3085c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSIN: 3095c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FCOS: 3105c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FPOWI: 3115c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FPOW: 3125c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG: 3135c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG2: 3145c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG10: 3155c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FEXP: 3165c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FEXP2: 3175c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FCEIL: 3185c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FTRUNC: 3195c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FRINT: 3205c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FNEARBYINT: 32141418d17cced656f91038b2482bc9d173b4974b0Hal Finkel case ISD::FROUND: 3225c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FFLOOR: 323846ce8ea67362d8b6d93ebae66f23e3c68dce9dfEli Friedman case ISD::FP_ROUND: 32443147afd71f6da4e7369a4ab9c681e5b4e0cf8c7Eli Friedman case ISD::FP_EXTEND: 3256b1e1d8b3d8d5a1b299d3c2897db9bf122b02c00Craig Topper case ISD::FMA: 326d0f3ef807ee4210b97a7a6bc4231e89403145b83Nadav Rotem case ISD::SIGN_EXTEND_INREG: 327c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::ANY_EXTEND_VECTOR_INREG: 328c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SIGN_EXTEND_VECTOR_INREG: 329c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::ZERO_EXTEND_VECTOR_INREG: 3306948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SMIN: 3316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::SMAX: 3326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::UMIN: 3336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::UMAX: 334556929a84bb8842cb07bebf4df67810d17be096eEli Friedman QueryType = Node->getValueType(0); 335556929a84bb8842cb07bebf4df67810d17be096eEli Friedman break; 336d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::FP_ROUND_INREG: 337d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 338d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman break; 339556929a84bb8842cb07bebf4df67810d17be096eEli Friedman case ISD::SINT_TO_FP: 340556929a84bb8842cb07bebf4df67810d17be096eEli Friedman case ISD::UINT_TO_FP: 341556929a84bb8842cb07bebf4df67810d17be096eEli Friedman QueryType = Node->getOperand(0).getValueType(); 3425c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case ISD::MSCATTER: 3446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType(); 3456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar break; 346cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::MSTORE: 347cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar QueryType = cast<MaskedStoreSDNode>(Node)->getValue().getValueType(); 348cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar break; 3495c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3505c22c8074404797f1313b1334757254fb5c6487aEli Friedman 351556929a84bb8842cb07bebf4df67810d17be096eEli Friedman switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) { 352cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar default: llvm_unreachable("This action is not supported yet!"); 3535c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Promote: 354c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Result = Promote(Op); 355c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Changed = true; 356c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines break; 357c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case TargetLowering::Legal: 3585c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3595c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Custom: { 3605c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue Tmp1 = TLI.LowerOperation(Op, DAG); 3615c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Tmp1.getNode()) { 3625c22c8074404797f1313b1334757254fb5c6487aEli Friedman Result = Tmp1; 3635c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3645c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3655c22c8074404797f1313b1334757254fb5c6487aEli Friedman // FALL THROUGH 3665c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3675c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Expand: 368c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Result = Expand(Op); 3695c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3705c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3715c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Make sure that the generated code is itself legal. 3725c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Result != Op) { 3735c22c8074404797f1313b1334757254fb5c6487aEli Friedman Result = LegalizeOp(Result); 3745c22c8074404797f1313b1334757254fb5c6487aEli Friedman Changed = true; 3755c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3765c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3775c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Note that LegalizeOp may be reentered even from single-use nodes, which 3785c22c8074404797f1313b1334757254fb5c6487aEli Friedman // means that we always must cache transformed nodes. 3795c22c8074404797f1313b1334757254fb5c6487aEli Friedman AddLegalizedOperand(Op, Result); 3805c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Result; 3815c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 3825c22c8074404797f1313b1334757254fb5c6487aEli Friedman 383c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::Promote(SDValue Op) { 384c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // For a few operations there is a specific concept for promotion based on 385c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // the operand's type. 386c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines switch (Op.getOpcode()) { 387c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SINT_TO_FP: 388c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::UINT_TO_FP: 389c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // "Promote" the operation by extending the operand. 390c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return PromoteINT_TO_FP(Op); 391c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::FP_TO_UINT: 392c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::FP_TO_SINT: 393c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Promote the operation by extending the operand. 394c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return PromoteFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT); 395c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 396c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 39737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // There are currently two cases of vector promotion: 39837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 1) Bitcasting a vector of integers to a different type to a vector of the 3990c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64. 4000c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // 2) Extending a vector of floats to a vector of the same number of larger 40137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32. 402319bb399233d3ee67233aa29235c8ad2148fb77dPatrik Hagglund MVT VT = Op.getSimpleValueType(); 4035c22c8074404797f1313b1334757254fb5c6487aEli Friedman assert(Op.getNode()->getNumValues() == 1 && 4045c22c8074404797f1313b1334757254fb5c6487aEli Friedman "Can't promote a vector with multiple results!"); 405319bb399233d3ee67233aa29235c8ad2148fb77dPatrik Hagglund MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); 406e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 4075c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 4085c22c8074404797f1313b1334757254fb5c6487aEli Friedman 4095c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 4105c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Op.getOperand(j).getValueType().isVector()) 41137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Op.getOperand(j) 41237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines .getValueType() 41337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines .getVectorElementType() 414ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .isFloatingPoint() && 415ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()) 41637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op.getOperand(j)); 41737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 41837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Op.getOperand(j)); 4195c22c8074404797f1313b1334757254fb5c6487aEli Friedman else 4205c22c8074404797f1313b1334757254fb5c6487aEli Friedman Operands[j] = Op.getOperand(j); 4215c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 4225c22c8074404797f1313b1334757254fb5c6487aEli Friedman 423cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar Op = DAG.getNode(Op.getOpcode(), dl, NVT, Operands, Op.getNode()->getFlags()); 424ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) || 425ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (VT.isVector() && VT.getVectorElementType().isFloatingPoint() && 426ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())) 4276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl)); 42837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 42937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(ISD::BITCAST, dl, VT, Op); 4305c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 4315c22c8074404797f1313b1334757254fb5c6487aEli Friedman 432c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) { 433926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // INT_TO_FP operations may require the input operand be promoted even 434926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // when the type is otherwise legal. 435926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach EVT VT = Op.getOperand(0).getValueType(); 436926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach assert(Op.getNode()->getNumValues() == 1 && 437926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach "Can't promote a vector with multiple results!"); 438926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 439926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // Normal getTypeToPromoteTo() doesn't work here, as that will promote 440926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // by widening the vector w/ the same element width and twice the number 441926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // of elements. We want the other way around, the same number of elements, 442926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // each twice the width. 443926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // 444926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // Increase the bitwidth of the element to the next pow-of-two 445926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // (which is greater than 8 bits). 446926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT NVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 44836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(NVT.isSimple() && "Promoting to a non-simple vector type!"); 449e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 450926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 451926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 452926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach unsigned Opc = Op.getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : 453926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach ISD::SIGN_EXTEND; 454926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 455926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach if (Op.getOperand(j).getValueType().isVector()) 456926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach Operands[j] = DAG.getNode(Opc, dl, NVT, Op.getOperand(j)); 457926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach else 458926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach Operands[j] = Op.getOperand(j); 459926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach } 460926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Op.getOpcode(), dl, Op.getValueType(), Operands); 462926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach} 463926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 46436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// For FP_TO_INT we promote the result type to a vector type with wider 46536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// elements and then truncate the result. This is different from the default 46636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// PromoteVector which uses bitcast to promote thus assumning that the 46736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// promoted vector type has the same overall size. 468c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op, bool isSigned) { 46936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op.getNode()->getNumValues() == 1 && 47036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "Can't promote a vector with multiple results!"); 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = Op.getValueType(); 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT NewVT; 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NewOpc; 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (1) { 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(NewVT.isSimple() && "Promoting to a non-simple vector type!"); 47836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) { 47936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewOpc = ISD::FP_TO_SINT; 48036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 48136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 48236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) { 48336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewOpc = ISD::FP_TO_UINT; 48436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 48536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 48636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 48736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc loc(Op); 48936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue promoted = DAG.getNode(NewOpc, SDLoc(Op), NewVT, Op.getOperand(0)); 49036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, promoted); 49136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 49236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 493e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 494e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav RotemSDValue VectorLegalizer::ExpandLoad(SDValue Op) { 495e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 496e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 497e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Chain = LD->getChain(); 498e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue BasePTR = LD->getBasePtr(); 499e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT SrcVT = LD->getMemoryVT(); 500fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem ISD::LoadExtType ExtType = LD->getExtensionType(); 501e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 502eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SmallVector<SDValue, 8> Vals; 503e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SmallVector<SDValue, 8> LoadChains; 504e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned NumElem = SrcVT.getVectorNumElements(); 505e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 506eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT SrcEltVT = SrcVT.getScalarType(); 507eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT DstEltVT = Op.getNode()->getValueType(0).getScalarType(); 508eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 509eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) { 510eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // When elements in a vector is not byte-addressable, we cannot directly 511eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // load each element by advancing pointer, which could only address bytes. 512eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // Instead, we load all significant words, mask bits off, and concatenate 513eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // them to form each element. Finally, they are extended to destination 514eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // scalar type to build the destination vector. 515cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar EVT WideVT = TLI.getPointerTy(DAG.getDataLayout()); 516eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 517eedff3547de6428798d0bd62c40fba3f93820922Michael Liao assert(WideVT.isRound() && 518eedff3547de6428798d0bd62c40fba3f93820922Michael Liao "Could not handle the sophisticated case when the widest integer is" 519eedff3547de6428798d0bd62c40fba3f93820922Michael Liao " not power of 2."); 520eedff3547de6428798d0bd62c40fba3f93820922Michael Liao assert(WideVT.bitsGE(SrcEltVT) && 521eedff3547de6428798d0bd62c40fba3f93820922Michael Liao "Type is not legalized?"); 522eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 523eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideBytes = WideVT.getStoreSize(); 524eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned Offset = 0; 525eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned RemainingBytes = SrcVT.getStoreSize(); 526eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SmallVector<SDValue, 8> LoadVals; 527eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 528eedff3547de6428798d0bd62c40fba3f93820922Michael Liao while (RemainingBytes > 0) { 529eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue ScalarLoad; 530eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned LoadBytes = WideBytes; 531eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 532eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (RemainingBytes >= LoadBytes) { 533eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR, 534eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->getPointerInfo().getWithOffset(Offset), 535eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->isVolatile(), LD->isNonTemporal(), 536ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LD->isInvariant(), 537ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MinAlign(LD->getAlignment(), Offset), 53837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LD->getAAInfo()); 539eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } else { 540eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT LoadVT = WideVT; 541eedff3547de6428798d0bd62c40fba3f93820922Michael Liao while (RemainingBytes < LoadBytes) { 542eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadBytes >>= 1; // Reduce the load size by half. 543eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3); 544eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 545eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR, 546eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->getPointerInfo().getWithOffset(Offset), 547eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVT, LD->isVolatile(), 54837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LD->isNonTemporal(), LD->isInvariant(), 549ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MinAlign(LD->getAlignment(), Offset), 550ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LD->getAAInfo()); 551eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 552eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 553eedff3547de6428798d0bd62c40fba3f93820922Michael Liao RemainingBytes -= LoadBytes; 554eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Offset += LoadBytes; 555eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 5566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(LoadBytes, dl, 5576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar BasePTR.getValueType())); 558eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 559eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVals.push_back(ScalarLoad.getValue(0)); 560eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadChains.push_back(ScalarLoad.getValue(1)); 561eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 562e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 563eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // Extract bits, pack and extend/trunc them into destination type. 564eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned SrcEltBits = SrcEltVT.getSizeInBits(); 5656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT); 566fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 567eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned BitOffset = 0; 568eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideIdx = 0; 569eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideBits = WideVT.getSizeInBits(); 570eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 571eedff3547de6428798d0bd62c40fba3f93820922Michael Liao for (unsigned Idx = 0; Idx != NumElem; ++Idx) { 572eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue Lo, Hi, ShAmt; 573eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 574eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (BitOffset < WideBits) { 575cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ShAmt = DAG.getConstant( 576cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); 577eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); 578eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); 579eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 580eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 581eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BitOffset += SrcEltBits; 582eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (BitOffset >= WideBits) { 583eedff3547de6428798d0bd62c40fba3f93820922Michael Liao WideIdx++; 584ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BitOffset -= WideBits; 585ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (BitOffset > 0) { 586cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ShAmt = DAG.getConstant( 587cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SrcEltBits - BitOffset, dl, 588cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); 589eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); 590eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask); 591eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 592eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 593eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 594eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (Hi.getNode()) 595eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); 596eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 597eedff3547de6428798d0bd62c40fba3f93820922Michael Liao switch (ExtType) { 598eedff3547de6428798d0bd62c40fba3f93820922Michael Liao default: llvm_unreachable("Unknown extended-load op!"); 599eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::EXTLOAD: 600eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT); 601eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 602eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::ZEXTLOAD: 603eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT); 604eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 605eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::SEXTLOAD: 606cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ShAmt = 607cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar DAG.getConstant(WideBits - SrcEltBits, dl, 608cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); 609eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt); 610eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt); 611eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT); 612eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 613eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 614eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Vals.push_back(Lo); 615eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 616eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } else { 617eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; 618eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 619eedff3547de6428798d0bd62c40fba3f93820922Michael Liao for (unsigned Idx=0; Idx<NumElem; Idx++) { 620eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl, 621eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Op.getNode()->getValueType(0).getScalarType(), 622eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), 623eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SrcVT.getScalarType(), 62437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LD->isVolatile(), LD->isNonTemporal(), LD->isInvariant(), 625ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MinAlign(LD->getAlignment(), Idx * Stride), LD->getAAInfo()); 626eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 627eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 6286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Stride, dl, BasePTR.getValueType())); 629eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 630eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Vals.push_back(ScalarLoad.getValue(0)); 631eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadChains.push_back(ScalarLoad.getValue(1)); 632eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 633e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 634fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 636e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, 637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getNode()->getValueType(0), Vals); 638e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 639e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op.getValue(0), Value); 640e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op.getValue(1), NewChain); 641e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 642e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return (Op.getResNo() ? NewChain : Value); 643e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem} 644e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 645e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav RotemSDValue VectorLegalizer::ExpandStore(SDValue Op) { 646e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 647e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 648e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Chain = ST->getChain(); 649e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue BasePTR = ST->getBasePtr(); 650e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Value = ST->getValue(); 651e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT StVT = ST->getMemoryVT(); 652e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 653e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned Alignment = ST->getAlignment(); 654e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem bool isVolatile = ST->isVolatile(); 655e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem bool isNonTemporal = ST->isNonTemporal(); 65637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AAMDNodes AAInfo = ST->getAAInfo(); 657e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 658e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned NumElem = StVT.getVectorNumElements(); 659e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // The type of the data we want to save 660e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT RegVT = Value.getValueType(); 661e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT RegSclVT = RegVT.getScalarType(); 662e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // The type of data as saved in memory. 663e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT MemSclVT = StVT.getScalarType(); 664e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 665e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Cast floats into integers 666e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned ScalarSize = MemSclVT.getSizeInBits(); 667e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 668e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Round odd types to the next pow of two. 669e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (!isPowerOf2_32(ScalarSize)) 670e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ScalarSize = NextPowerOf2(ScalarSize); 671e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 672e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Store Stride in bytes 673e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned Stride = ScalarSize/8; 674e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Extract each of the elements from the original vector 675e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // and save them into memory individually. 676e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SmallVector<SDValue, 8> Stores; 677e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem for (unsigned Idx = 0; Idx < NumElem; Idx++) { 678cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SDValue Ex = DAG.getNode( 679cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ISD::EXTRACT_VECTOR_ELT, dl, RegSclVT, Value, 680cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 681e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 682e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // This scalar TruncStore may be illegal, but we legalize it later. 683e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR, 684e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, 685ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines isVolatile, isNonTemporal, MinAlign(Alignment, Idx*Stride), 686ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AAInfo); 687e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 688fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 6896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Stride, dl, BasePTR.getValueType())); 690fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 691e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Stores.push_back(Store); 692e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 694e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op, TF); 695e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TF; 696e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem} 697e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 698c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::Expand(SDValue Op) { 699c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines switch (Op->getOpcode()) { 700c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SIGN_EXTEND_INREG: 701c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandSEXTINREG(Op); 702c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::ANY_EXTEND_VECTOR_INREG: 703c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandANY_EXTEND_VECTOR_INREG(Op); 704c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SIGN_EXTEND_VECTOR_INREG: 705c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandSIGN_EXTEND_VECTOR_INREG(Op); 706c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::ZERO_EXTEND_VECTOR_INREG: 707c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandZERO_EXTEND_VECTOR_INREG(Op); 708c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::BSWAP: 709c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandBSWAP(Op); 710c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::VSELECT: 711c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandVSELECT(Op); 712c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SELECT: 713c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandSELECT(Op); 714c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::UINT_TO_FP: 715c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandUINT_TO_FLOAT(Op); 716c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::FNEG: 717c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return ExpandFNEG(Op); 718c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case ISD::SETCC: 719c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return UnrollVSETCC(Op); 720cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar case ISD::BITREVERSE: 721cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return ExpandBITREVERSE(Op); 722c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines default: 723c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.UnrollVectorOp(Op.getNode()); 724c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 725c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 726c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 727e757f00446fb3c80a96d729f0530b87e9148db7fNadav RotemSDValue VectorLegalizer::ExpandSELECT(SDValue Op) { 728e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Lower a select instruction where the condition is a scalar and the 729e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // operands are vectors. Lower this select to VSELECT and implement it 730155615d7dc4e4fbfd3b7273720a76356468edf46Stephen Lin // using XOR AND OR. The selector bit is broadcasted. 731e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem EVT VT = Op.getValueType(); 732e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 733e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 734e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Mask = Op.getOperand(0); 735e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Op1 = Op.getOperand(1); 736e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Op2 = Op.getOperand(2); 737e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 738e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem assert(VT.isVector() && !Mask.getValueType().isVector() 739e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem && Op1.getValueType() == Op2.getValueType() && "Invalid type"); 740e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 741e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem unsigned NumElem = VT.getVectorNumElements(); 742e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 743e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // If we can't even use the basic vector operations of 744e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // AND,OR,XOR, we will have to scalarize the op. 745e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Notice that the operation may be 'promoted' which means that it is 746e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // 'bitcasted' to another type which is handled. 747e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Also, we need to be able to construct a splat vector using BUILD_VECTOR. 748e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 749e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 750e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 751e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::BUILD_VECTOR, VT) == TargetLowering::Expand) 752e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 753e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 754e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Generate a mask operand. 755c6c08508cab2fc4f52e9ab44dce272e240661e06Matt Arsenault EVT MaskTy = VT.changeVectorElementTypeToInteger(); 756e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 757e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // What is the size of each element in the vector mask. 758e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem EVT BitTy = MaskTy.getScalarType(); 759e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 760b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault Mask = DAG.getSelect(DL, BitTy, Mask, 7616948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, 7626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar BitTy), 7636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, BitTy)); 764e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 765e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Broadcast the mask so that the entire vector is all-one or all zero. 766e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SmallVector<SDValue, 8> Ops(NumElem, Mask); 767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskTy, Ops); 768e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 769e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Bitcast the operands to be the same type as the mask. 770e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // This is needed when we select between FP types because 771e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // the mask is a vector of integers. 772e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1); 773e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2); 774e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 775e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue AllOnes = DAG.getConstant( 7766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, MaskTy); 777e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes); 778e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 779e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask); 780e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask); 781e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2); 782e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 783e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem} 784e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 78566de2af815f97e484c1940ff157ffbb809931b20Nadav RotemSDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) { 78666de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem EVT VT = Op.getValueType(); 78766de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 7884dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer // Make sure that the SRA and SHL instructions are available. 78966de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || 7904dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) 79166de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 79266de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 793e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 79466de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT(); 79566de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 79666de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem unsigned BW = VT.getScalarType().getSizeInBits(); 79766de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem unsigned OrigBW = OrigTy.getScalarType().getSizeInBits(); 7986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT); 79966de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 80066de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem Op = Op.getOperand(0); 8014dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz); 80266de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); 80366de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem} 80466de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 805c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// Generically expand a vector anyext in register to a shuffle of the relevant 806c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// lanes into the appropriate locations, with other lanes left undef. 807c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) { 808c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 809c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 810c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int NumElements = VT.getVectorNumElements(); 811c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Src = Op.getOperand(0); 812c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT SrcVT = Src.getValueType(); 813c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int NumSrcElements = SrcVT.getVectorNumElements(); 814c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 815c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Build a base mask of undef shuffles. 816c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<int, 16> ShuffleMask; 817c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShuffleMask.resize(NumSrcElements, -1); 818c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 819c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Place the extended lanes into the correct locations. 820c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int ExtLaneScale = NumSrcElements / NumElements; 821cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0; 822c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (int i = 0; i < NumElements; ++i) 823c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShuffleMask[i * ExtLaneScale + EndianOffset] = i; 824c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 825c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode( 826c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ISD::BITCAST, DL, VT, 827c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask)); 828c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 829c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 830c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) { 831c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 832c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 833c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Src = Op.getOperand(0); 834c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT SrcVT = Src.getValueType(); 835c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 836c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // First build an any-extend node which can be legalized above when we 837c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // recurse through it. 838c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Op = DAG.getAnyExtendVectorInReg(Src, DL, VT); 839c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 840c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Now we need sign extend. Do this by shifting the elements. Even if these 841c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // aren't legal operations, they have a better chance of being legalized 842c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // without full scalarization than the sign extension does. 843c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned EltWidth = VT.getVectorElementType().getSizeInBits(); 844c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned SrcEltWidth = SrcVT.getVectorElementType().getSizeInBits(); 8456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT); 846c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(ISD::SRA, DL, VT, 847c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount), 848c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShiftAmount); 849c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 850c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 851c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// Generically expand a vector zext in register to a shuffle of the relevant 852c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// lanes into the appropriate locations, a blend of zero into the high bits, 853c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// and a bitcast to the wider element type. 854c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) { 855c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc DL(Op); 856c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 857c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int NumElements = VT.getVectorNumElements(); 858c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Src = Op.getOperand(0); 859c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT SrcVT = Src.getValueType(); 860c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int NumSrcElements = SrcVT.getVectorNumElements(); 861c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 862c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Build up a zero vector to blend into this one. 863c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT SrcScalarVT = SrcVT.getScalarType(); 8646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue ScalarZero = DAG.getTargetConstant(0, DL, SrcScalarVT); 865c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<SDValue, 4> BuildVectorOperands(NumSrcElements, ScalarZero); 866c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, DL, SrcVT, BuildVectorOperands); 867c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 868c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Shuffle the incoming lanes into the correct position, and pull all other 869c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // lanes from the zero vector. 870c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<int, 16> ShuffleMask; 871c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShuffleMask.reserve(NumSrcElements); 872c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (int i = 0; i < NumSrcElements; ++i) 873c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShuffleMask.push_back(i); 874c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 875c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines int ExtLaneScale = NumSrcElements / NumElements; 876cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0; 877c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (int i = 0; i < NumElements; ++i) 878c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i; 879c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 880c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(ISD::BITCAST, DL, VT, 881c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask)); 882c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 883c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue VectorLegalizer::ExpandBSWAP(SDValue Op) { 885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Generate a byte wise shuffle mask for the BSWAP. 888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<int, 16> ShuffleMask; 889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8; 890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) 891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int J = ScalarSizeInBytes - 1; J >= 0; --J) 892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShuffleMask.push_back((I * ScalarSizeInBytes) + J); 893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size()); 895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only emit a shuffle if the mask is legal. 897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) 898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.UnrollVectorOp(Op.getNode()); 899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Op.getOperand(0)); 902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), 903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShuffleMask.data()); 904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BITCAST, DL, VT, Op); 905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 907cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga NainarSDValue VectorLegalizer::ExpandBITREVERSE(SDValue Op) { 908cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar EVT VT = Op.getValueType(); 909cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 910cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // If we have the scalar operation, it's probably cheaper to unroll it. 911cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) 912cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return DAG.UnrollVectorOp(Op.getNode()); 913cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 914cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // If we have the appropriate vector bit operations, it is better to use them 915cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // than unrolling and expanding each component. 916cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar if (!TLI.isOperationLegalOrCustom(ISD::SHL, VT) || 917cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar !TLI.isOperationLegalOrCustom(ISD::SRL, VT) || 918cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar !TLI.isOperationLegalOrCustom(ISD::AND, VT) || 919cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar !TLI.isOperationLegalOrCustom(ISD::OR, VT)) 920cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return DAG.UnrollVectorOp(Op.getNode()); 921cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 922cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // Let LegalizeDAG handle this later. 923cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar return Op; 924cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar} 925cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar 926aec5861bb6ace3734163c000cb75ca2e22e29caaNadav RotemSDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { 927aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // Implement VSELECT in terms of XOR, AND, OR 928aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // on platforms which do not support blend natively. 929e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 930aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 931aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Mask = Op.getOperand(0); 932aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Op1 = Op.getOperand(1); 933aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Op2 = Op.getOperand(2); 934aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 935798925bac73b9320e7b32001478921910ac781d2Matt Arsenault EVT VT = Mask.getValueType(); 936798925bac73b9320e7b32001478921910ac781d2Matt Arsenault 937aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // If we can't even use the basic vector operations of 938aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // AND,OR,XOR, we will have to scalarize the op. 939815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem // Notice that the operation may be 'promoted' which means that it is 940815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem // 'bitcasted' to another type which is handled. 941d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // This operation also isn't safe with AND, OR, XOR when the boolean 942d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // type is 0/1 as we need an all ones vector constant to mask with. 943d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // FIXME: Sign extend 1 to all ones if thats legal on the target. 944815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 945815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 946c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 947c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines TLI.getBooleanContents(Op1.getValueType()) != 948c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines TargetLowering::ZeroOrNegativeOneBooleanContent) 949815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 95006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 951798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // If the mask and the type are different sizes, unroll the vector op. This 952798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // can occur when getSetCCResultType returns something that is different in 953798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8. 954798925bac73b9320e7b32001478921910ac781d2Matt Arsenault if (VT.getSizeInBits() != Op1.getValueType().getSizeInBits()) 955798925bac73b9320e7b32001478921910ac781d2Matt Arsenault return DAG.UnrollVectorOp(Op.getNode()); 956798925bac73b9320e7b32001478921910ac781d2Matt Arsenault 957aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // Bitcast the operands to be the same type as the mask. 958aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // This is needed when we select between FP types because 959aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // the mask is a vector of integers. 960aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); 961aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); 96206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 963aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue AllOnes = DAG.getConstant( 9646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), DL, VT); 965aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes); 966aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 967aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); 968aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); 9693ab32ea49ec13182f1397dc89c37551692f67140Nadav Rotem SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2); 9703ab32ea49ec13182f1397dc89c37551692f67140Nadav Rotem return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 971aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem} 972aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 973aec5861bb6ace3734163c000cb75ca2e22e29caaNadav RotemSDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) { 97406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem EVT VT = Op.getOperand(0).getValueType(); 975e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 97606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 97706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Make sure that the SINT_TO_FP and SRL instructions are available. 978815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem if (TLI.getOperationAction(ISD::SINT_TO_FP, VT) == TargetLowering::Expand || 979815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) 980815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 98106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 98206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem EVT SVT = VT.getScalarType(); 98306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem assert((SVT.getSizeInBits() == 64 || SVT.getSizeInBits() == 32) && 98406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide"); 98506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 98606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem unsigned BW = SVT.getSizeInBits(); 9876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue HalfWord = DAG.getConstant(BW/2, DL, VT); 98806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 98906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Constants to clear the upper part of the word. 99006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Notice that we can also use SHL+SHR, but using a constant is slightly 99106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // faster on x86. 99206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem uint64_t HWMask = (SVT.getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF; 9936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT); 99406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 99506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Two to the power of half-word-size. 9966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue TWOHW = DAG.getConstantFP(1 << (BW/2), DL, Op.getValueType()); 99706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 99806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Clear upper part of LO, lower HI 99906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord); 100006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask); 100106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 100206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Convert hi and lo to floats 100306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Convert the hi part back to the upper values 1004cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // TODO: Can any fast-math-flags be set on these nodes? 100506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI); 100606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem fHI = DAG.getNode(ISD::FMUL, DL, Op.getValueType(), fHI, TWOHW); 100706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO); 100806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 100906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Add the two halves 101006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem return DAG.getNode(ISD::FADD, DL, Op.getValueType(), fHI, fLO); 101106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem} 101206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 101306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 10145c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::ExpandFNEG(SDValue Op) { 10155c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) { 10166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 10176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Zero = DAG.getConstantFP(-0.0, DL, Op.getValueType()); 1018cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB. 10196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::FSUB, DL, Op.getValueType(), 10205c22c8074404797f1313b1334757254fb5c6487aEli Friedman Zero, Op.getOperand(0)); 10215c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 1022cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang return DAG.UnrollVectorOp(Op.getNode()); 10235c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 10245c22c8074404797f1313b1334757254fb5c6487aEli Friedman 10255c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { 1026e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = Op.getValueType(); 10275c22c8074404797f1313b1334757254fb5c6487aEli Friedman unsigned NumElems = VT.getVectorNumElements(); 1028e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VT.getVectorElementType(); 10295c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2); 1030e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 1031e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 10325c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 8> Ops(NumElems); 10335c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned i = 0; i < NumElems; ++i) { 1034cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SDValue LHSElem = DAG.getNode( 1035cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 1036cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 1037cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar SDValue RHSElem = DAG.getNode( 1038cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 1039cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 1040225ed7069caae9ece32d8bd3d15c6e41e21cc04bMatt Arsenault Ops[i] = DAG.getNode(ISD::SETCC, dl, 1041cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar TLI.getSetCCResultType(DAG.getDataLayout(), 1042cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar *DAG.getContext(), TmpEltVT), 10435c22c8074404797f1313b1334757254fb5c6487aEli Friedman LHSElem, RHSElem, CC); 1044b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], 1045b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault DAG.getConstant(APInt::getAllOnesValue 10466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar (EltVT.getSizeInBits()), dl, EltVT), 10476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, EltVT)); 10485c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 1049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 10505c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 10515c22c8074404797f1313b1334757254fb5c6487aEli Friedman 10525c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 10535c22c8074404797f1313b1334757254fb5c6487aEli Friedman 10545c22c8074404797f1313b1334757254fb5c6487aEli Friedmanbool SelectionDAG::LegalizeVectors() { 10555c22c8074404797f1313b1334757254fb5c6487aEli Friedman return VectorLegalizer(*this).Run(); 10565c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 1057