15c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===-- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ---===// 25c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 35c22c8074404797f1313b1334757254fb5c6487aEli Friedman// The LLVM Compiler Infrastructure 45c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 55c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This file is distributed under the University of Illinois Open Source 65c22c8074404797f1313b1334757254fb5c6487aEli Friedman// License. See LICENSE.TXT for details. 75c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 85c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===----------------------------------------------------------------------===// 95c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 105c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This file implements the SelectionDAG::LegalizeVectors method. 115c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 125c22c8074404797f1313b1334757254fb5c6487aEli Friedman// The vector legalizer looks for vector operations which might need to be 13509150f973ae650a57b79010a3ec36e60e40f41dEli Friedman// scalarized and legalizes them. This is a separate step from Legalize because 14509150f973ae650a57b79010a3ec36e60e40f41dEli Friedman// scalarizing can introduce illegal types. For example, suppose we have an 155c22c8074404797f1313b1334757254fb5c6487aEli Friedman// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition 165c22c8074404797f1313b1334757254fb5c6487aEli Friedman// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the 175c22c8074404797f1313b1334757254fb5c6487aEli Friedman// operation, which introduces nodes with the illegal type i64 which must be 185c22c8074404797f1313b1334757254fb5c6487aEli Friedman// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC; 195c22c8074404797f1313b1334757254fb5c6487aEli Friedman// the operation must be unrolled, which introduces nodes with the illegal 205c22c8074404797f1313b1334757254fb5c6487aEli Friedman// type i8 which must be promoted. 215c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 225c22c8074404797f1313b1334757254fb5c6487aEli Friedman// This does not legalize vector manipulations like ISD::BUILD_VECTOR, 2398ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman// or operations that happen to take a vector which are custom-lowered; 2498ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman// the legalization for such operations never produces nodes 255c22c8074404797f1313b1334757254fb5c6487aEli Friedman// with illegal types, so it's okay to put off legalizing them until 265c22c8074404797f1313b1334757254fb5c6487aEli Friedman// SelectionDAG::Legalize runs. 275c22c8074404797f1313b1334757254fb5c6487aEli Friedman// 285c22c8074404797f1313b1334757254fb5c6487aEli Friedman//===----------------------------------------------------------------------===// 295c22c8074404797f1313b1334757254fb5c6487aEli Friedman 305c22c8074404797f1313b1334757254fb5c6487aEli Friedman#include "llvm/CodeGen/SelectionDAG.h" 315c22c8074404797f1313b1334757254fb5c6487aEli Friedman#include "llvm/Target/TargetLowering.h" 325c22c8074404797f1313b1334757254fb5c6487aEli Friedmanusing namespace llvm; 335c22c8074404797f1313b1334757254fb5c6487aEli Friedman 345c22c8074404797f1313b1334757254fb5c6487aEli Friedmannamespace { 355c22c8074404797f1313b1334757254fb5c6487aEli Friedmanclass VectorLegalizer { 365c22c8074404797f1313b1334757254fb5c6487aEli Friedman SelectionDAG& DAG; 37d858e90f039f5fcdc2fa93035e911a5a9505cc50Dan Gohman const TargetLowering &TLI; 385c22c8074404797f1313b1334757254fb5c6487aEli Friedman bool Changed; // Keep track of whether anything changed 395c22c8074404797f1313b1334757254fb5c6487aEli Friedman 40cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// For nodes that are of legal width, and that have more than one use, this 41cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// map indicates what regularized operand to use. This allows us to avoid 42cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// legalizing the same thing more than once. 43ea387fc3b8cf12c3c6ad218b81eca156e8173bbaPreston Gurd SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes; 445c22c8074404797f1313b1334757254fb5c6487aEli Friedman 45cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Adds a node to the translation cache. 465c22c8074404797f1313b1334757254fb5c6487aEli Friedman void AddLegalizedOperand(SDValue From, SDValue To) { 475c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.insert(std::make_pair(From, To)); 485c22c8074404797f1313b1334757254fb5c6487aEli Friedman // If someone requests legalization of the new node, return itself. 495c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (From != To) 505c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.insert(std::make_pair(To, To)); 515c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 525c22c8074404797f1313b1334757254fb5c6487aEli Friedman 53cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Legalizes the given node. 545c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue LegalizeOp(SDValue Op); 55cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 56cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Assuming the node is legal, "legalize" the results. 575c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue TranslateLegalizeResults(SDValue Op, SDValue Result); 58cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 59cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implements unrolling a VSETCC. 605c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue UnrollVSETCC(SDValue Op); 61cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 62cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement expand-based legalization of vector operations. 63cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 64cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// This is just a high-level routine to dispatch to specific code paths for 65cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// operations to legalize them. 66cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Expand(SDValue Op); 67cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 68cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implements expansion for FNEG; falls back to UnrollVectorOp if 69cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// FSUB isn't legal. 70cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 71cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if 72cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// SINT_TO_FLOAT and SHR on vectors isn't legal. 7306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue ExpandUINT_TO_FLOAT(SDValue Op); 74cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 75cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement expansion for SIGN_EXTEND_INREG using SRL and SRA. 7666de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem SDValue ExpandSEXTINREG(SDValue Op); 77cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 78cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement expansion for ANY_EXTEND_VECTOR_INREG. 79cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 80cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// Shuffles the low lanes of the operand into place and bitcasts to the proper 81cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// type. The contents of the bits in the extended part of each element are 82cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// undef. 83cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op); 84cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 85cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement expansion for SIGN_EXTEND_VECTOR_INREG. 86cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 87cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// Shuffles the low lanes of the operand into place, bitcasts to the proper 88cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// type, then shifts left and arithmetic shifts right to introduce a sign 89cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// extension. 90cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op); 91cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 92cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement expansion for ZERO_EXTEND_VECTOR_INREG. 93cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 94cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// Shuffles the low lanes of the operand into place and blends zeros into 95cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// the remaining lanes, finally bitcasting to the proper type. 96cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op); 97cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 98cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Expand bswap of vectors into a shuffle if legal. 99dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ExpandBSWAP(SDValue Op); 100cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 101cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implement vselect in terms of XOR, AND, OR when blend is not 102cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// supported by the target. 103aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue ExpandVSELECT(SDValue Op); 104e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue ExpandSELECT(SDValue Op); 105e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue ExpandLoad(SDValue Op); 106e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue ExpandStore(SDValue Op); 1075c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue ExpandFNEG(SDValue Op); 108cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 109cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implements vector promotion. 110cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 111cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// This is essentially just bitcasting the operands to a different type and 112cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// bitcasting the result back to the original type. 113cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Promote(SDValue Op); 114cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 115cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implements [SU]INT_TO_FP vector promotion. 116cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 117cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// This is a [zs]ext of the input operand to the next size up. 118cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue PromoteINT_TO_FP(SDValue Op); 119cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 120cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Implements FP_TO_[SU]INT vector promotion of the result type. 121cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// 122cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// It is promoted to the next size up integer type. The result is then 123cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// truncated back to the original type. 124cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue PromoteFP_TO_INT(SDValue Op, bool isSigned); 125cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 126cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hinespublic: 127cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines /// \brief Begin legalizer the vector operations in the DAG. 1285c22c8074404797f1313b1334757254fb5c6487aEli Friedman bool Run(); 1295c22c8074404797f1313b1334757254fb5c6487aEli Friedman VectorLegalizer(SelectionDAG& dag) : 1305c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(false) {} 1315c22c8074404797f1313b1334757254fb5c6487aEli Friedman}; 1325c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1335c22c8074404797f1313b1334757254fb5c6487aEli Friedmanbool VectorLegalizer::Run() { 134d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // Before we start legalizing vector nodes, check if there are any vectors. 135d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem bool HasVectors = false; 136d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 13736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) { 138d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // Check if the values of the nodes contain vectors. We don't need to check 139d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // the operands because we are going to check their values at some point. 140d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem for (SDNode::value_iterator J = I->value_begin(), E = I->value_end(); 141d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem J != E; ++J) 142d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem HasVectors |= J->isVector(); 143d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 144d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // If we found a vector node we can start the legalization. 145d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem if (HasVectors) 146d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem break; 147d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem } 148d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 149d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem // If this basic block has no vectors then no need to legalize vectors. 150d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem if (!HasVectors) 151d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem return false; 152d99a5a3ab4d47c6532bcf17a01677b1730599057Nadav Rotem 1535c22c8074404797f1313b1334757254fb5c6487aEli Friedman // The legalize process is inherently a bottom-up recursive process (users 1545c22c8074404797f1313b1334757254fb5c6487aEli Friedman // legalize their uses before themselves). Given infinite stack space, we 1555c22c8074404797f1313b1334757254fb5c6487aEli Friedman // could just start legalizing on the root and traverse the whole graph. In 1565c22c8074404797f1313b1334757254fb5c6487aEli Friedman // practice however, this causes us to run out of stack space on large basic 1575c22c8074404797f1313b1334757254fb5c6487aEli Friedman // blocks. To avoid this problem, compute an ordering of the nodes where each 1585c22c8074404797f1313b1334757254fb5c6487aEli Friedman // node is only legalized after all of its operands are legalized. 1595c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.AssignTopologicalOrder(); 1605c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 16136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) 1625c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizeOp(SDValue(I, 0)); 1635c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1645c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Finally, it's possible the root changed. Get the new root. 1655c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue OldRoot = DAG.getRoot(); 1665c22c8074404797f1313b1334757254fb5c6487aEli Friedman assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 1675c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.setRoot(LegalizedNodes[OldRoot]); 1685c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1695c22c8074404797f1313b1334757254fb5c6487aEli Friedman LegalizedNodes.clear(); 1705c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1715c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Remove dead nodes now. 1725c22c8074404797f1313b1334757254fb5c6487aEli Friedman DAG.RemoveDeadNodes(); 1735c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1745c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Changed; 1755c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 1765c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1775c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) { 1785c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Generic legalization: just pass the operand through. 1795c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned i = 0, e = Op.getNode()->getNumValues(); i != e; ++i) 1805c22c8074404797f1313b1334757254fb5c6487aEli Friedman AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 1815c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Result.getValue(Op.getResNo()); 1825c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 1835c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1845c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::LegalizeOp(SDValue Op) { 1855c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Note that LegalizeOp may be reentered even from single-use nodes, which 1865c22c8074404797f1313b1334757254fb5c6487aEli Friedman // means that we always must cache transformed nodes. 1875c22c8074404797f1313b1334757254fb5c6487aEli Friedman DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1885c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (I != LegalizedNodes.end()) return I->second; 1895c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1905c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDNode* Node = Op.getNode(); 1915c22c8074404797f1313b1334757254fb5c6487aEli Friedman 1925c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Legalize the operands 1935c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 8> Ops; 1945c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1955c22c8074404797f1313b1334757254fb5c6487aEli Friedman Ops.push_back(LegalizeOp(Node->getOperand(i))); 1965c22c8074404797f1313b1334757254fb5c6487aEli Friedman 197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0); 1985c22c8074404797f1313b1334757254fb5c6487aEli Friedman 199e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (Op.getOpcode() == ISD::LOAD) { 200e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 201e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ISD::LoadExtType ExtType = LD->getExtensionType(); 202e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) { 203e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (TLI.isLoadExtLegal(LD->getExtensionType(), LD->getMemoryVT())) 204e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TranslateLegalizeResults(Op, Result); 205e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Changed = true; 206e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return LegalizeOp(ExpandLoad(Op)); 207e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 208e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } else if (Op.getOpcode() == ISD::STORE) { 209e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 210e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT StVT = ST->getMemoryVT(); 21188ef514cc63c3f22f78eaf4dd295d349b4070819Patrik Hagglund MVT ValVT = ST->getValue().getSimpleValueType(); 212e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (StVT.isVector() && ST->isTruncatingStore()) 21388ef514cc63c3f22f78eaf4dd295d349b4070819Patrik Hagglund switch (TLI.getTruncStoreAction(ValVT, StVT.getSimpleVT())) { 2145e25ee8a1fcf8288d00d731b0f7ab7976f33b123Craig Topper default: llvm_unreachable("This action is not supported yet!"); 215e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem case TargetLowering::Legal: 216e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TranslateLegalizeResults(Op, Result); 217e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem case TargetLowering::Custom: 218e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Changed = true; 219d00968a7a52b23a0d9fc5c1c4b07aceb2cd15006Tom Stellard return TranslateLegalizeResults(Op, TLI.LowerOperation(Result, DAG)); 220e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem case TargetLowering::Expand: 221e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Changed = true; 222e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return LegalizeOp(ExpandStore(Op)); 223e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 224e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 225e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 2265c22c8074404797f1313b1334757254fb5c6487aEli Friedman bool HasVectorValue = false; 2275c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end(); 2285c22c8074404797f1313b1334757254fb5c6487aEli Friedman J != E; 2295c22c8074404797f1313b1334757254fb5c6487aEli Friedman ++J) 2305c22c8074404797f1313b1334757254fb5c6487aEli Friedman HasVectorValue |= J->isVector(); 2315c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (!HasVectorValue) 2325c22c8074404797f1313b1334757254fb5c6487aEli Friedman return TranslateLegalizeResults(Op, Result); 2335c22c8074404797f1313b1334757254fb5c6487aEli Friedman 234e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT QueryType; 2355c22c8074404797f1313b1334757254fb5c6487aEli Friedman switch (Op.getOpcode()) { 2365c22c8074404797f1313b1334757254fb5c6487aEli Friedman default: 2375c22c8074404797f1313b1334757254fb5c6487aEli Friedman return TranslateLegalizeResults(Op, Result); 2385c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ADD: 2395c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SUB: 2405c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::MUL: 2415c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SDIV: 2425c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::UDIV: 2435c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SREM: 2445c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::UREM: 2455c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FADD: 2465c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSUB: 2475c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FMUL: 2485c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FDIV: 2495c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FREM: 2505c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::AND: 2515c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::OR: 2525c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::XOR: 2535c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SHL: 2545c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SRA: 2555c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SRL: 2565c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ROTL: 2575c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ROTR: 25836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case ISD::BSWAP: 2595c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::CTLZ: 26063974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTTZ: 26163974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTLZ_ZERO_UNDEF: 26263974b2144c87c962effdc0508c27643c8ad98b6Chandler Carruth case ISD::CTTZ_ZERO_UNDEF: 2635c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::CTPOP: 2645c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SELECT: 265aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem case ISD::VSELECT: 2665c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SELECT_CC: 26728b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands case ISD::SETCC: 2685c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ZERO_EXTEND: 2695c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::ANY_EXTEND: 2705c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::TRUNCATE: 2715c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::SIGN_EXTEND: 2725c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FP_TO_SINT: 2735c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FP_TO_UINT: 2745c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FNEG: 2755c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FABS: 27666d1fa6f4b443ac9f8bcea5d1f71a73ada733a42Hal Finkel case ISD::FCOPYSIGN: 2775c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSQRT: 2785c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FSIN: 2795c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FCOS: 2805c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FPOWI: 2815c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FPOW: 2825c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG: 2835c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG2: 2845c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FLOG10: 2855c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FEXP: 2865c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FEXP2: 2875c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FCEIL: 2885c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FTRUNC: 2895c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FRINT: 2905c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FNEARBYINT: 29141418d17cced656f91038b2482bc9d173b4974b0Hal Finkel case ISD::FROUND: 2925c22c8074404797f1313b1334757254fb5c6487aEli Friedman case ISD::FFLOOR: 293846ce8ea67362d8b6d93ebae66f23e3c68dce9dfEli Friedman case ISD::FP_ROUND: 29443147afd71f6da4e7369a4ab9c681e5b4e0cf8c7Eli Friedman case ISD::FP_EXTEND: 2956b1e1d8b3d8d5a1b299d3c2897db9bf122b02c00Craig Topper case ISD::FMA: 296d0f3ef807ee4210b97a7a6bc4231e89403145b83Nadav Rotem case ISD::SIGN_EXTEND_INREG: 297cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::ANY_EXTEND_VECTOR_INREG: 298cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SIGN_EXTEND_VECTOR_INREG: 299cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::ZERO_EXTEND_VECTOR_INREG: 300556929a84bb8842cb07bebf4df67810d17be096eEli Friedman QueryType = Node->getValueType(0); 301556929a84bb8842cb07bebf4df67810d17be096eEli Friedman break; 302d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman case ISD::FP_ROUND_INREG: 303d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 304d1996360399ad6dbe75ee185b661b16c83146373Dan Gohman break; 305556929a84bb8842cb07bebf4df67810d17be096eEli Friedman case ISD::SINT_TO_FP: 306556929a84bb8842cb07bebf4df67810d17be096eEli Friedman case ISD::UINT_TO_FP: 307556929a84bb8842cb07bebf4df67810d17be096eEli Friedman QueryType = Node->getOperand(0).getValueType(); 3085c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3095c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3105c22c8074404797f1313b1334757254fb5c6487aEli Friedman 311556929a84bb8842cb07bebf4df67810d17be096eEli Friedman switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) { 3125c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Promote: 313cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Result = Promote(Op); 314cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Changed = true; 315cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines break; 316cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case TargetLowering::Legal: 3175c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3185c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Custom: { 3195c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue Tmp1 = TLI.LowerOperation(Op, DAG); 3205c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Tmp1.getNode()) { 3215c22c8074404797f1313b1334757254fb5c6487aEli Friedman Result = Tmp1; 3225c22c8074404797f1313b1334757254fb5c6487aEli Friedman break; 3235c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3245c22c8074404797f1313b1334757254fb5c6487aEli Friedman // FALL THROUGH 3255c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3265c22c8074404797f1313b1334757254fb5c6487aEli Friedman case TargetLowering::Expand: 327cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Result = Expand(Op); 3285c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3295c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3305c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Make sure that the generated code is itself legal. 3315c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Result != Op) { 3325c22c8074404797f1313b1334757254fb5c6487aEli Friedman Result = LegalizeOp(Result); 3335c22c8074404797f1313b1334757254fb5c6487aEli Friedman Changed = true; 3345c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3355c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3365c22c8074404797f1313b1334757254fb5c6487aEli Friedman // Note that LegalizeOp may be reentered even from single-use nodes, which 3375c22c8074404797f1313b1334757254fb5c6487aEli Friedman // means that we always must cache transformed nodes. 3385c22c8074404797f1313b1334757254fb5c6487aEli Friedman AddLegalizedOperand(Op, Result); 3395c22c8074404797f1313b1334757254fb5c6487aEli Friedman return Result; 3405c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 3415c22c8074404797f1313b1334757254fb5c6487aEli Friedman 342cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::Promote(SDValue Op) { 343cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // For a few operations there is a specific concept for promotion based on 344cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // the operand's type. 345cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines switch (Op.getOpcode()) { 346cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SINT_TO_FP: 347cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::UINT_TO_FP: 348cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // "Promote" the operation by extending the operand. 349cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return PromoteINT_TO_FP(Op); 350cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FP_TO_UINT: 351cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FP_TO_SINT: 352cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Promote the operation by extending the operand. 353cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return PromoteFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT); 354cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 355cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 356cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // The rest of the time, vector "promotion" is basically just bitcasting and 357cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // doing the operation in a different type. For example, x86 promotes 358cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // ISD::AND on v2i32 to v1i64. 359319bb399233d3ee67233aa29235c8ad2148fb77dPatrik Hagglund MVT VT = Op.getSimpleValueType(); 3605c22c8074404797f1313b1334757254fb5c6487aEli Friedman assert(Op.getNode()->getNumValues() == 1 && 3615c22c8074404797f1313b1334757254fb5c6487aEli Friedman "Can't promote a vector with multiple results!"); 362319bb399233d3ee67233aa29235c8ad2148fb77dPatrik Hagglund MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); 363e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 3645c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 3655c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3665c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 3675c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (Op.getOperand(j).getValueType().isVector()) 368bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Op.getOperand(j)); 3695c22c8074404797f1313b1334757254fb5c6487aEli Friedman else 3705c22c8074404797f1313b1334757254fb5c6487aEli Friedman Operands[j] = Op.getOperand(j); 3715c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3725c22c8074404797f1313b1334757254fb5c6487aEli Friedman 373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = DAG.getNode(Op.getOpcode(), dl, NVT, Operands); 3745c22c8074404797f1313b1334757254fb5c6487aEli Friedman 375bf17cfa3f904e488e898ac2e3af706fd1a892f08Wesley Peck return DAG.getNode(ISD::BITCAST, dl, VT, Op); 3765c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 3775c22c8074404797f1313b1334757254fb5c6487aEli Friedman 378cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) { 379926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // INT_TO_FP operations may require the input operand be promoted even 380926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // when the type is otherwise legal. 381926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach EVT VT = Op.getOperand(0).getValueType(); 382926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach assert(Op.getNode()->getNumValues() == 1 && 383926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach "Can't promote a vector with multiple results!"); 384926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 385926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // Normal getTypeToPromoteTo() doesn't work here, as that will promote 386926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // by widening the vector w/ the same element width and twice the number 387926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // of elements. We want the other way around, the same number of elements, 388926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // each twice the width. 389926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // 390926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // Increase the bitwidth of the element to the next pow-of-two 391926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach // (which is greater than 8 bits). 392926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 39336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT NVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 39436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(NVT.isSimple() && "Promoting to a non-simple vector type!"); 395e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 396926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 397926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 398926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach unsigned Opc = Op.getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : 399926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach ISD::SIGN_EXTEND; 400926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 401926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach if (Op.getOperand(j).getValueType().isVector()) 402926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach Operands[j] = DAG.getNode(Opc, dl, NVT, Op.getOperand(j)); 403926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach else 404926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach Operands[j] = Op.getOperand(j); 405926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach } 406926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Op.getOpcode(), dl, Op.getValueType(), Operands); 408926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach} 409926dc168c893e1848a18e45fd78f9b99d6f4cd5dJim Grosbach 41036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// For FP_TO_INT we promote the result type to a vector type with wider 41136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// elements and then truncate the result. This is different from the default 41236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// PromoteVector which uses bitcast to promote thus assumning that the 41336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// promoted vector type has the same overall size. 414cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op, bool isSigned) { 41536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op.getNode()->getNumValues() == 1 && 41636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "Can't promote a vector with multiple results!"); 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = Op.getValueType(); 41836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT NewVT; 42036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NewOpc; 42136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (1) { 42236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 42336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(NewVT.isSimple() && "Promoting to a non-simple vector type!"); 42436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) { 42536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewOpc = ISD::FP_TO_SINT; 42636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 42736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 42836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) { 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewOpc = ISD::FP_TO_UINT; 43036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 43136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 43236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 43336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc loc(Op); 43536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDValue promoted = DAG.getNode(NewOpc, SDLoc(Op), NewVT, Op.getOperand(0)); 43636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, promoted); 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 439e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 440e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav RotemSDValue VectorLegalizer::ExpandLoad(SDValue Op) { 441e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 442e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 443e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Chain = LD->getChain(); 444e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue BasePTR = LD->getBasePtr(); 445e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT SrcVT = LD->getMemoryVT(); 446fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem ISD::LoadExtType ExtType = LD->getExtensionType(); 447e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 448eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SmallVector<SDValue, 8> Vals; 449e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SmallVector<SDValue, 8> LoadChains; 450e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned NumElem = SrcVT.getVectorNumElements(); 451e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 452eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT SrcEltVT = SrcVT.getScalarType(); 453eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT DstEltVT = Op.getNode()->getValueType(0).getScalarType(); 454eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 455eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) { 456eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // When elements in a vector is not byte-addressable, we cannot directly 457eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // load each element by advancing pointer, which could only address bytes. 458eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // Instead, we load all significant words, mask bits off, and concatenate 459eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // them to form each element. Finally, they are extended to destination 460eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // scalar type to build the destination vector. 461eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT WideVT = TLI.getPointerTy(); 462eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 463eedff3547de6428798d0bd62c40fba3f93820922Michael Liao assert(WideVT.isRound() && 464eedff3547de6428798d0bd62c40fba3f93820922Michael Liao "Could not handle the sophisticated case when the widest integer is" 465eedff3547de6428798d0bd62c40fba3f93820922Michael Liao " not power of 2."); 466eedff3547de6428798d0bd62c40fba3f93820922Michael Liao assert(WideVT.bitsGE(SrcEltVT) && 467eedff3547de6428798d0bd62c40fba3f93820922Michael Liao "Type is not legalized?"); 468eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 469eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideBytes = WideVT.getStoreSize(); 470eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned Offset = 0; 471eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned RemainingBytes = SrcVT.getStoreSize(); 472eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SmallVector<SDValue, 8> LoadVals; 473eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 474eedff3547de6428798d0bd62c40fba3f93820922Michael Liao while (RemainingBytes > 0) { 475eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue ScalarLoad; 476eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned LoadBytes = WideBytes; 477eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 478eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (RemainingBytes >= LoadBytes) { 479eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR, 480eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->getPointerInfo().getWithOffset(Offset), 481eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->isVolatile(), LD->isNonTemporal(), 48266589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford LD->isInvariant(), LD->getAlignment(), 48366589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford LD->getTBAAInfo()); 484eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } else { 485eedff3547de6428798d0bd62c40fba3f93820922Michael Liao EVT LoadVT = WideVT; 486eedff3547de6428798d0bd62c40fba3f93820922Michael Liao while (RemainingBytes < LoadBytes) { 487eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadBytes >>= 1; // Reduce the load size by half. 488eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3); 489eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 490eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR, 491eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->getPointerInfo().getWithOffset(Offset), 492eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVT, LD->isVolatile(), 49366589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford LD->isNonTemporal(), LD->getAlignment(), 49466589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford LD->getTBAAInfo()); 495eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 496eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 497eedff3547de6428798d0bd62c40fba3f93820922Michael Liao RemainingBytes -= LoadBytes; 498eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Offset += LoadBytes; 499eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 500edd08f74289c6ba3b3f8e730e4ab825ef9bd6492Tom Stellard DAG.getConstant(LoadBytes, BasePTR.getValueType())); 501eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 502eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadVals.push_back(ScalarLoad.getValue(0)); 503eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadChains.push_back(ScalarLoad.getValue(1)); 504eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 505e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 506eedff3547de6428798d0bd62c40fba3f93820922Michael Liao // Extract bits, pack and extend/trunc them into destination type. 507eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned SrcEltBits = SrcEltVT.getSizeInBits(); 508eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, WideVT); 509fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 510eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned BitOffset = 0; 511eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideIdx = 0; 512eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned WideBits = WideVT.getSizeInBits(); 513eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 514eedff3547de6428798d0bd62c40fba3f93820922Michael Liao for (unsigned Idx = 0; Idx != NumElem; ++Idx) { 515eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue Lo, Hi, ShAmt; 516eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 517eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (BitOffset < WideBits) { 518eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ShAmt = DAG.getConstant(BitOffset, TLI.getShiftAmountTy(WideVT)); 519eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); 520eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); 521eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 522eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 523eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BitOffset += SrcEltBits; 524eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (BitOffset >= WideBits) { 525eedff3547de6428798d0bd62c40fba3f93820922Michael Liao WideIdx++; 526eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Offset -= WideBits; 527eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (Offset > 0) { 528eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ShAmt = DAG.getConstant(SrcEltBits - Offset, 529eedff3547de6428798d0bd62c40fba3f93820922Michael Liao TLI.getShiftAmountTy(WideVT)); 530eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); 531eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask); 532eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 533eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 534eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 535eedff3547de6428798d0bd62c40fba3f93820922Michael Liao if (Hi.getNode()) 536eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); 537eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 538eedff3547de6428798d0bd62c40fba3f93820922Michael Liao switch (ExtType) { 539eedff3547de6428798d0bd62c40fba3f93820922Michael Liao default: llvm_unreachable("Unknown extended-load op!"); 540eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::EXTLOAD: 541eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT); 542eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 543eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::ZEXTLOAD: 544eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT); 545eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 546eedff3547de6428798d0bd62c40fba3f93820922Michael Liao case ISD::SEXTLOAD: 547eedff3547de6428798d0bd62c40fba3f93820922Michael Liao ShAmt = DAG.getConstant(WideBits - SrcEltBits, 548eedff3547de6428798d0bd62c40fba3f93820922Michael Liao TLI.getShiftAmountTy(WideVT)); 549eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt); 550eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt); 551eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT); 552eedff3547de6428798d0bd62c40fba3f93820922Michael Liao break; 553eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 554eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Vals.push_back(Lo); 555eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 556eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } else { 557eedff3547de6428798d0bd62c40fba3f93820922Michael Liao unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; 558eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 559eedff3547de6428798d0bd62c40fba3f93820922Michael Liao for (unsigned Idx=0; Idx<NumElem; Idx++) { 560eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl, 561eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Op.getNode()->getValueType(0).getScalarType(), 562eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), 563eedff3547de6428798d0bd62c40fba3f93820922Michael Liao SrcVT.getScalarType(), 564eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LD->isVolatile(), LD->isNonTemporal(), 56566589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford LD->getAlignment(), LD->getTBAAInfo()); 566eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 567eedff3547de6428798d0bd62c40fba3f93820922Michael Liao BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 568edd08f74289c6ba3b3f8e730e4ab825ef9bd6492Tom Stellard DAG.getConstant(Stride, BasePTR.getValueType())); 569eedff3547de6428798d0bd62c40fba3f93820922Michael Liao 570eedff3547de6428798d0bd62c40fba3f93820922Michael Liao Vals.push_back(ScalarLoad.getValue(0)); 571eedff3547de6428798d0bd62c40fba3f93820922Michael Liao LoadChains.push_back(ScalarLoad.getValue(1)); 572eedff3547de6428798d0bd62c40fba3f93820922Michael Liao } 573e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 574fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 575dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 576e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, 577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getNode()->getValueType(0), Vals); 578e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 579e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op.getValue(0), Value); 580e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op.getValue(1), NewChain); 581e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 582e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return (Op.getResNo() ? NewChain : Value); 583e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem} 584e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 585e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav RotemSDValue VectorLegalizer::ExpandStore(SDValue Op) { 586e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 587e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 588e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Chain = ST->getChain(); 589e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue BasePTR = ST->getBasePtr(); 590e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Value = ST->getValue(); 591e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT StVT = ST->getMemoryVT(); 592e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 593e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned Alignment = ST->getAlignment(); 594e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem bool isVolatile = ST->isVolatile(); 595e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem bool isNonTemporal = ST->isNonTemporal(); 59666589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford const MDNode *TBAAInfo = ST->getTBAAInfo(); 597e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 598e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned NumElem = StVT.getVectorNumElements(); 599e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // The type of the data we want to save 600e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT RegVT = Value.getValueType(); 601e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT RegSclVT = RegVT.getScalarType(); 602e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // The type of data as saved in memory. 603e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem EVT MemSclVT = StVT.getScalarType(); 604e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 605e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Cast floats into integers 606e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned ScalarSize = MemSclVT.getSizeInBits(); 607e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 608e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Round odd types to the next pow of two. 609e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem if (!isPowerOf2_32(ScalarSize)) 610e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ScalarSize = NextPowerOf2(ScalarSize); 611e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 612e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Store Stride in bytes 613e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem unsigned Stride = ScalarSize/8; 614e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // Extract each of the elements from the original vector 615e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // and save them into memory individually. 616e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SmallVector<SDValue, 8> Stores; 617e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem for (unsigned Idx = 0; Idx < NumElem; Idx++) { 618e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 619425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard RegSclVT, Value, DAG.getConstant(Idx, TLI.getVectorIdxTy())); 620e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 621e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem // This scalar TruncStore may be illegal, but we legalize it later. 622e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR, 623e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, 62466589dcc8fb5dcf0894a9a80a8dee890a4f3a379Richard Sandiford isVolatile, isNonTemporal, Alignment, TBAAInfo); 625e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 626fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 627edd08f74289c6ba3b3f8e730e4ab825ef9bd6492Tom Stellard DAG.getConstant(Stride, BasePTR.getValueType())); 628fbf19ef1860e33b202ff73a269b8b0bf9157460eNadav Rotem 629e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem Stores.push_back(Store); 630e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem } 631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 632e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem AddLegalizedOperand(Op, TF); 633e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem return TF; 634e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem} 635e9b58d0aac4e89b53a4be0e6f289b66649e1512bNadav Rotem 636cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::Expand(SDValue Op) { 637cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines switch (Op->getOpcode()) { 638cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SIGN_EXTEND_INREG: 639cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandSEXTINREG(Op); 640cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::ANY_EXTEND_VECTOR_INREG: 641cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandANY_EXTEND_VECTOR_INREG(Op); 642cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SIGN_EXTEND_VECTOR_INREG: 643cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandSIGN_EXTEND_VECTOR_INREG(Op); 644cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::ZERO_EXTEND_VECTOR_INREG: 645cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandZERO_EXTEND_VECTOR_INREG(Op); 646cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::BSWAP: 647cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandBSWAP(Op); 648cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::VSELECT: 649cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandVSELECT(Op); 650cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SELECT: 651cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandSELECT(Op); 652cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::UINT_TO_FP: 653cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandUINT_TO_FLOAT(Op); 654cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::FNEG: 655cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return ExpandFNEG(Op); 656cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines case ISD::SETCC: 657cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return UnrollVSETCC(Op); 658cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines default: 659cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.UnrollVectorOp(Op.getNode()); 660cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 661cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 662cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 663e757f00446fb3c80a96d729f0530b87e9148db7fNadav RotemSDValue VectorLegalizer::ExpandSELECT(SDValue Op) { 664e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Lower a select instruction where the condition is a scalar and the 665e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // operands are vectors. Lower this select to VSELECT and implement it 666155615d7dc4e4fbfd3b7273720a76356468edf46Stephen Lin // using XOR AND OR. The selector bit is broadcasted. 667e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem EVT VT = Op.getValueType(); 668e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 669e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 670e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Mask = Op.getOperand(0); 671e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Op1 = Op.getOperand(1); 672e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Op2 = Op.getOperand(2); 673e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 674e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem assert(VT.isVector() && !Mask.getValueType().isVector() 675e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem && Op1.getValueType() == Op2.getValueType() && "Invalid type"); 676e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 677e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem unsigned NumElem = VT.getVectorNumElements(); 678e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 679e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // If we can't even use the basic vector operations of 680e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // AND,OR,XOR, we will have to scalarize the op. 681e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Notice that the operation may be 'promoted' which means that it is 682e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // 'bitcasted' to another type which is handled. 683e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Also, we need to be able to construct a splat vector using BUILD_VECTOR. 684e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 685e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 686e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 687e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem TLI.getOperationAction(ISD::BUILD_VECTOR, VT) == TargetLowering::Expand) 688e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 689e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 690e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Generate a mask operand. 691c6c08508cab2fc4f52e9ab44dce272e240661e06Matt Arsenault EVT MaskTy = VT.changeVectorElementTypeToInteger(); 692e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 693e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // What is the size of each element in the vector mask. 694e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem EVT BitTy = MaskTy.getScalarType(); 695e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 696b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault Mask = DAG.getSelect(DL, BitTy, Mask, 697f55ef64544f8ba81e50154a98f144f7b7783ed40Nadav Rotem DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), BitTy), 698ee77da6b28d7f56e4e531ae8b4f8093de760a2e5Nadav Rotem DAG.getConstant(0, BitTy)); 699e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 700e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Broadcast the mask so that the entire vector is all-one or all zero. 701e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SmallVector<SDValue, 8> Ops(NumElem, Mask); 702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskTy, Ops); 703e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 704e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // Bitcast the operands to be the same type as the mask. 705e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // This is needed when we select between FP types because 706e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem // the mask is a vector of integers. 707e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1); 708e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2); 709e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 710e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue AllOnes = DAG.getConstant( 711e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem APInt::getAllOnesValue(BitTy.getSizeInBits()), MaskTy); 712e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes); 713e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 714e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask); 715e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask); 716e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2); 717e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 718e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem} 719e757f00446fb3c80a96d729f0530b87e9148db7fNadav Rotem 72066de2af815f97e484c1940ff157ffbb809931b20Nadav RotemSDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) { 72166de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem EVT VT = Op.getValueType(); 72266de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 7234dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer // Make sure that the SRA and SHL instructions are available. 72466de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || 7254dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) 72666de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 72766de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 728e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 72966de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT(); 73066de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 73166de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem unsigned BW = VT.getScalarType().getSizeInBits(); 73266de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem unsigned OrigBW = OrigTy.getScalarType().getSizeInBits(); 73366de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem SDValue ShiftSz = DAG.getConstant(BW - OrigBW, VT); 73466de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 73566de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem Op = Op.getOperand(0); 7364dc478308f0de13d9ce20915193ac8c3318c5bd6Benjamin Kramer Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz); 73766de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); 73866de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem} 73966de2af815f97e484c1940ff157ffbb809931b20Nadav Rotem 740cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// Generically expand a vector anyext in register to a shuffle of the relevant 741cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// lanes into the appropriate locations, with other lanes left undef. 742cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) { 743cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 744cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 745cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int NumElements = VT.getVectorNumElements(); 746cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 747cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SrcVT = Src.getValueType(); 748cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int NumSrcElements = SrcVT.getVectorNumElements(); 749cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 750cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Build a base mask of undef shuffles. 751cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SmallVector<int, 16> ShuffleMask; 752cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShuffleMask.resize(NumSrcElements, -1); 753cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 754cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Place the extended lanes into the correct locations. 755cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int ExtLaneScale = NumSrcElements / NumElements; 756cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int EndianOffset = TLI.isBigEndian() ? ExtLaneScale - 1 : 0; 757cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (int i = 0; i < NumElements; ++i) 758cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShuffleMask[i * ExtLaneScale + EndianOffset] = i; 759cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 760cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode( 761cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ISD::BITCAST, DL, VT, 762cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask)); 763cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 764cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 765cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) { 766cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 767cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 768cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 769cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SrcVT = Src.getValueType(); 770cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 771cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // First build an any-extend node which can be legalized above when we 772cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // recurse through it. 773cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines Op = DAG.getAnyExtendVectorInReg(Src, DL, VT); 774cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 775cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Now we need sign extend. Do this by shifting the elements. Even if these 776cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // aren't legal operations, they have a better chance of being legalized 777cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // without full scalarization than the sign extension does. 778cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines unsigned EltWidth = VT.getVectorElementType().getSizeInBits(); 779cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines unsigned SrcEltWidth = SrcVT.getVectorElementType().getSizeInBits(); 780cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, VT); 781cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::SRA, DL, VT, 782cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount), 783cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShiftAmount); 784cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 785cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 786cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// Generically expand a vector zext in register to a shuffle of the relevant 787cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// lanes into the appropriate locations, a blend of zero into the high bits, 788cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines// and a bitcast to the wider element type. 789cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesSDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) { 790cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDLoc DL(Op); 791cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT VT = Op.getValueType(); 792cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int NumElements = VT.getVectorNumElements(); 793cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Src = Op.getOperand(0); 794cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SrcVT = Src.getValueType(); 795cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int NumSrcElements = SrcVT.getVectorNumElements(); 796cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 797cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Build up a zero vector to blend into this one. 798cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines EVT SrcScalarVT = SrcVT.getScalarType(); 799cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue ScalarZero = DAG.getTargetConstant(0, SrcScalarVT); 800cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SmallVector<SDValue, 4> BuildVectorOperands(NumSrcElements, ScalarZero); 801cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, DL, SrcVT, BuildVectorOperands); 802cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 803cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // Shuffle the incoming lanes into the correct position, and pull all other 804cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines // lanes from the zero vector. 805cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines SmallVector<int, 16> ShuffleMask; 806cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShuffleMask.reserve(NumSrcElements); 807cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (int i = 0; i < NumSrcElements; ++i) 808cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShuffleMask.push_back(i); 809cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 810cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int ExtLaneScale = NumSrcElements / NumElements; 811cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines int EndianOffset = TLI.isBigEndian() ? ExtLaneScale - 1 : 0; 812cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines for (int i = 0; i < NumElements; ++i) 813cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i; 814cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 815cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return DAG.getNode(ISD::BITCAST, DL, VT, 816cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask)); 817cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines} 818cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines 819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue VectorLegalizer::ExpandBSWAP(SDValue Op) { 820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Generate a byte wise shuffle mask for the BSWAP. 823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<int, 16> ShuffleMask; 824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8; 825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) 826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int J = ScalarSizeInBytes - 1; J >= 0; --J) 827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShuffleMask.push_back((I * ScalarSizeInBytes) + J); 828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size()); 830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only emit a shuffle if the mask is legal. 832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) 833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.UnrollVectorOp(Op.getNode()); 834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Op.getOperand(0)); 837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), 838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShuffleMask.data()); 839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BITCAST, DL, VT, Op); 840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 842aec5861bb6ace3734163c000cb75ca2e22e29caaNadav RotemSDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { 843aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // Implement VSELECT in terms of XOR, AND, OR 844aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // on platforms which do not support blend natively. 845e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 846aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 847aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Mask = Op.getOperand(0); 848aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Op1 = Op.getOperand(1); 849aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue Op2 = Op.getOperand(2); 850aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 851798925bac73b9320e7b32001478921910ac781d2Matt Arsenault EVT VT = Mask.getValueType(); 852798925bac73b9320e7b32001478921910ac781d2Matt Arsenault 853aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // If we can't even use the basic vector operations of 854aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // AND,OR,XOR, we will have to scalarize the op. 855815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem // Notice that the operation may be 'promoted' which means that it is 856815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem // 'bitcasted' to another type which is handled. 857d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // This operation also isn't safe with AND, OR, XOR when the boolean 858d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // type is 0/1 as we need an all ones vector constant to mask with. 859d906017c1a8b5c7c49f1bc21c13e8b85306298b8Pete Cooper // FIXME: Sign extend 1 to all ones if thats legal on the target. 860815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 861815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 862cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 863cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines TLI.getBooleanContents(Op1.getValueType()) != 864cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines TargetLowering::ZeroOrNegativeOneBooleanContent) 865815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 86606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 867798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // If the mask and the type are different sizes, unroll the vector op. This 868798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // can occur when getSetCCResultType returns something that is different in 869798925bac73b9320e7b32001478921910ac781d2Matt Arsenault // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8. 870798925bac73b9320e7b32001478921910ac781d2Matt Arsenault if (VT.getSizeInBits() != Op1.getValueType().getSizeInBits()) 871798925bac73b9320e7b32001478921910ac781d2Matt Arsenault return DAG.UnrollVectorOp(Op.getNode()); 872798925bac73b9320e7b32001478921910ac781d2Matt Arsenault 873aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // Bitcast the operands to be the same type as the mask. 874aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // This is needed when we select between FP types because 875aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem // the mask is a vector of integers. 876aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); 877aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); 87806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 879aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue AllOnes = DAG.getConstant( 880aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), VT); 881aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes); 882aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 883aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); 884aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); 8853ab32ea49ec13182f1397dc89c37551692f67140Nadav Rotem SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2); 8863ab32ea49ec13182f1397dc89c37551692f67140Nadav Rotem return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 887aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem} 888aec5861bb6ace3734163c000cb75ca2e22e29caaNadav Rotem 889aec5861bb6ace3734163c000cb75ca2e22e29caaNadav RotemSDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) { 89006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem EVT VT = Op.getOperand(0).getValueType(); 891e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc DL(Op); 89206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 89306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Make sure that the SINT_TO_FP and SRL instructions are available. 894815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem if (TLI.getOperationAction(ISD::SINT_TO_FP, VT) == TargetLowering::Expand || 895815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) 896815af82b74fa0901e818f5d16ee418675f399101Nadav Rotem return DAG.UnrollVectorOp(Op.getNode()); 89706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 89806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem EVT SVT = VT.getScalarType(); 89906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem assert((SVT.getSizeInBits() == 64 || SVT.getSizeInBits() == 32) && 90006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide"); 90106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 90206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem unsigned BW = SVT.getSizeInBits(); 90306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue HalfWord = DAG.getConstant(BW/2, VT); 90406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 90506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Constants to clear the upper part of the word. 90606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Notice that we can also use SHL+SHR, but using a constant is slightly 90706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // faster on x86. 90806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem uint64_t HWMask = (SVT.getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF; 90906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue HalfWordMask = DAG.getConstant(HWMask, VT); 91006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 91106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Two to the power of half-word-size. 91206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue TWOHW = DAG.getConstantFP((1<<(BW/2)), Op.getValueType()); 91306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 91406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Clear upper part of LO, lower HI 91506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord); 91606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask); 91706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 91806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Convert hi and lo to floats 91906cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Convert the hi part back to the upper values 92006cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI); 92106cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem fHI = DAG.getNode(ISD::FMUL, DL, Op.getValueType(), fHI, TWOHW); 92206cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO); 92306cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 92406cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem // Add the two halves 92506cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem return DAG.getNode(ISD::FADD, DL, Op.getValueType(), fHI, fLO); 92606cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem} 92706cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 92806cc324b9da1dc8fb7360a560343c28f5e7a940aNadav Rotem 9295c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::ExpandFNEG(SDValue Op) { 9305c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) { 9315c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue Zero = DAG.getConstantFP(-0.0, Op.getValueType()); 932ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 9335c22c8074404797f1313b1334757254fb5c6487aEli Friedman Zero, Op.getOperand(0)); 9345c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 935cd6e725f21852e2f8cdf5fd0e65eb42c224776f8Mon P Wang return DAG.UnrollVectorOp(Op.getNode()); 9365c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 9375c22c8074404797f1313b1334757254fb5c6487aEli Friedman 9385c22c8074404797f1313b1334757254fb5c6487aEli FriedmanSDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { 939e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = Op.getValueType(); 9405c22c8074404797f1313b1334757254fb5c6487aEli Friedman unsigned NumElems = VT.getVectorNumElements(); 941e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT EltVT = VT.getVectorElementType(); 9425c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2); 943e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 944e4fae84b0b13bd5d66cf619fd2108dbb6064395dBenjamin Kramer SDLoc dl(Op); 9455c22c8074404797f1313b1334757254fb5c6487aEli Friedman SmallVector<SDValue, 8> Ops(NumElems); 9465c22c8074404797f1313b1334757254fb5c6487aEli Friedman for (unsigned i = 0; i < NumElems; ++i) { 9475c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 948425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(i, TLI.getVectorIdxTy())); 9495c22c8074404797f1313b1334757254fb5c6487aEli Friedman SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 950425b76c2314ff7ee7ad507011bdda1988ae481efTom Stellard DAG.getConstant(i, TLI.getVectorIdxTy())); 951225ed7069caae9ece32d8bd3d15c6e41e21cc04bMatt Arsenault Ops[i] = DAG.getNode(ISD::SETCC, dl, 952225ed7069caae9ece32d8bd3d15c6e41e21cc04bMatt Arsenault TLI.getSetCCResultType(*DAG.getContext(), TmpEltVT), 9535c22c8074404797f1313b1334757254fb5c6487aEli Friedman LHSElem, RHSElem, CC); 954b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], 955b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault DAG.getConstant(APInt::getAllOnesValue 956b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault (EltVT.getSizeInBits()), EltVT), 957b05e4778f0871cbb02f61e4d55ad7375738a1d01Matt Arsenault DAG.getConstant(0, EltVT)); 9585c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 9605c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 9615c22c8074404797f1313b1334757254fb5c6487aEli Friedman 9625c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 9635c22c8074404797f1313b1334757254fb5c6487aEli Friedman 9645c22c8074404797f1313b1334757254fb5c6487aEli Friedmanbool SelectionDAG::LegalizeVectors() { 9655c22c8074404797f1313b1334757254fb5c6487aEli Friedman return VectorLegalizer(*this).Run(); 9665c22c8074404797f1313b1334757254fb5c6487aEli Friedman} 967